@effortlessmotion/pdf-to-md 0.1.8 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +5 -5
- package/dist/cjs/analyze.d.cts +3 -1
- package/dist/cjs/analyze.d.ts +3 -1
- package/dist/cjs/analyze.js +38 -100
- package/dist/cjs/analyze.js.map +1 -1
- package/dist/cjs/audit.d.cts +6 -5
- package/dist/cjs/audit.d.ts +6 -5
- package/dist/cjs/audit.js +368 -60
- package/dist/cjs/audit.js.map +1 -1
- package/dist/cjs/cli/extract.js +13 -48
- package/dist/cjs/cli/extract.js.map +1 -1
- package/dist/cjs/collect.js +91 -35
- package/dist/cjs/collect.js.map +1 -1
- package/dist/cjs/errors.d.cts +2 -0
- package/dist/cjs/errors.d.ts +2 -0
- package/dist/cjs/errors.js.map +1 -1
- package/dist/cjs/extract.d.cts +20 -9
- package/dist/cjs/extract.d.ts +20 -9
- package/dist/cjs/extract.js +333 -106
- package/dist/cjs/extract.js.map +1 -1
- package/dist/cjs/lines.js +21 -10
- package/dist/cjs/lines.js.map +1 -1
- package/dist/cjs/markdown.js +140 -54
- package/dist/cjs/markdown.js.map +1 -1
- package/dist/cjs/page-marker.d.cts +1 -1
- package/dist/cjs/page-marker.d.ts +1 -1
- package/dist/cjs/page-marker.js +39 -8
- package/dist/cjs/page-marker.js.map +1 -1
- package/dist/cjs/render-analysis.d.cts +4 -2
- package/dist/cjs/render-analysis.d.ts +4 -2
- package/dist/cjs/render-analysis.js +5 -3
- package/dist/cjs/render-analysis.js.map +1 -1
- package/dist/cjs/report.d.cts +2 -2
- package/dist/cjs/report.d.ts +2 -2
- package/dist/cjs/report.js +27 -8
- package/dist/cjs/report.js.map +1 -1
- package/dist/cjs/text-analysis.d.cts +12 -0
- package/dist/cjs/text-analysis.d.ts +12 -0
- package/dist/cjs/text-analysis.js +175 -0
- package/dist/cjs/text-analysis.js.map +1 -0
- package/dist/cjs/types.d.cts +40 -3
- package/dist/cjs/types.d.ts +40 -3
- package/dist/cjs/types.js +2 -3
- package/dist/cjs/types.js.map +1 -1
- package/dist/esm/analyze.d.ts +3 -1
- package/dist/esm/analyze.js +18 -89
- package/dist/esm/analyze.js.map +1 -1
- package/dist/esm/audit.d.ts +6 -5
- package/dist/esm/audit.js +236 -29
- package/dist/esm/audit.js.map +1 -1
- package/dist/esm/cli/extract.js +10 -23
- package/dist/esm/cli/extract.js.map +1 -1
- package/dist/esm/collect.js +64 -15
- package/dist/esm/collect.js.map +1 -1
- package/dist/esm/errors.d.ts +2 -0
- package/dist/esm/errors.js.map +1 -1
- package/dist/esm/extract.d.ts +20 -9
- package/dist/esm/extract.js +167 -76
- package/dist/esm/extract.js.map +1 -1
- package/dist/esm/lines.js +14 -10
- package/dist/esm/lines.js.map +1 -1
- package/dist/esm/markdown.js +110 -27
- package/dist/esm/markdown.js.map +1 -1
- package/dist/esm/page-marker.d.ts +1 -1
- package/dist/esm/page-marker.js +15 -9
- package/dist/esm/page-marker.js.map +1 -1
- package/dist/esm/render-analysis.d.ts +4 -2
- package/dist/esm/render-analysis.js +9 -5
- package/dist/esm/render-analysis.js.map +1 -1
- package/dist/esm/report.d.ts +2 -2
- package/dist/esm/report.js +12 -7
- package/dist/esm/report.js.map +1 -1
- package/dist/esm/text-analysis.d.ts +12 -0
- package/dist/esm/text-analysis.js +81 -0
- package/dist/esm/text-analysis.js.map +1 -0
- package/dist/esm/types.d.ts +40 -3
- package/dist/esm/types.js +4 -4
- package/dist/esm/types.js.map +1 -1
- package/package.json +1 -1
- package/skills/pdf-to-md/SKILL.md +39 -66
|
@@ -0,0 +1,175 @@
|
|
|
1
|
+
// Shared text-layer analysis. It keeps PDF source runs separate from layout
|
|
2
|
+
// reconstruction, while Markdown blocks retain the source-run indexes that
|
|
3
|
+
// produced them.
|
|
4
|
+
"use strict";
|
|
5
|
+
Object.defineProperty(exports, "__esModule", {
|
|
6
|
+
value: true
|
|
7
|
+
});
|
|
8
|
+
function _export(target, all) {
|
|
9
|
+
for(var name in all)Object.defineProperty(target, name, {
|
|
10
|
+
enumerable: true,
|
|
11
|
+
get: Object.getOwnPropertyDescriptor(all, name).get
|
|
12
|
+
});
|
|
13
|
+
}
|
|
14
|
+
_export(exports, {
|
|
15
|
+
get buildTextAnalysis () {
|
|
16
|
+
return buildTextAnalysis;
|
|
17
|
+
},
|
|
18
|
+
get selectCollectedPages () {
|
|
19
|
+
return selectCollectedPages;
|
|
20
|
+
}
|
|
21
|
+
});
|
|
22
|
+
var _linests = require("./lines.js");
|
|
23
|
+
var _markdownts = require("./markdown.js");
|
|
24
|
+
var _reportts = require("./report.js");
|
|
25
|
+
function _define_property(obj, key, value) {
|
|
26
|
+
if (key in obj) {
|
|
27
|
+
Object.defineProperty(obj, key, {
|
|
28
|
+
value: value,
|
|
29
|
+
enumerable: true,
|
|
30
|
+
configurable: true,
|
|
31
|
+
writable: true
|
|
32
|
+
});
|
|
33
|
+
} else obj[key] = value;
|
|
34
|
+
return obj;
|
|
35
|
+
}
|
|
36
|
+
function _object_spread(target) {
|
|
37
|
+
for(var i = 1; i < arguments.length; i++){
|
|
38
|
+
var source = arguments[i] != null ? arguments[i] : {};
|
|
39
|
+
var ownKeys = Object.keys(source);
|
|
40
|
+
if (typeof Object.getOwnPropertySymbols === "function") {
|
|
41
|
+
ownKeys = ownKeys.concat(Object.getOwnPropertySymbols(source).filter(function(sym) {
|
|
42
|
+
return Object.getOwnPropertyDescriptor(source, sym).enumerable;
|
|
43
|
+
}));
|
|
44
|
+
}
|
|
45
|
+
ownKeys.forEach(function(key) {
|
|
46
|
+
_define_property(target, key, source[key]);
|
|
47
|
+
});
|
|
48
|
+
}
|
|
49
|
+
return target;
|
|
50
|
+
}
|
|
51
|
+
function sourceWord(glyph, dims) {
|
|
52
|
+
return {
|
|
53
|
+
text: glyph.s,
|
|
54
|
+
box: {
|
|
55
|
+
x: dims.width > 0 ? glyph.bounds.x / dims.width : 0,
|
|
56
|
+
y: dims.height > 0 ? (dims.height - glyph.bounds.y - glyph.bounds.h) / dims.height : 0,
|
|
57
|
+
w: dims.width > 0 ? glyph.bounds.w / dims.width : 0,
|
|
58
|
+
h: dims.height > 0 ? glyph.bounds.h / dims.height : 0
|
|
59
|
+
}
|
|
60
|
+
};
|
|
61
|
+
}
|
|
62
|
+
function callbackWords(pages) {
|
|
63
|
+
return pages.flatMap(function(page) {
|
|
64
|
+
return page.words.map(function(word) {
|
|
65
|
+
return _object_spread({
|
|
66
|
+
page: page.page,
|
|
67
|
+
text: word.text
|
|
68
|
+
}, word.box);
|
|
69
|
+
});
|
|
70
|
+
});
|
|
71
|
+
}
|
|
72
|
+
function selectCollectedPages(collected, pageNumbers) {
|
|
73
|
+
var pages = collected.pages.filter(function(page) {
|
|
74
|
+
return pageNumbers.has(page.page);
|
|
75
|
+
});
|
|
76
|
+
var heightChars = new Map();
|
|
77
|
+
var _iteratorNormalCompletion = true, _didIteratorError = false, _iteratorError = undefined;
|
|
78
|
+
try {
|
|
79
|
+
for(var _iterator = pages[Symbol.iterator](), _step; !(_iteratorNormalCompletion = (_step = _iterator.next()).done); _iteratorNormalCompletion = true){
|
|
80
|
+
var page = _step.value;
|
|
81
|
+
var _heightChars_get;
|
|
82
|
+
var _iteratorNormalCompletion1 = true, _didIteratorError1 = false, _iteratorError1 = undefined;
|
|
83
|
+
try {
|
|
84
|
+
for(var _iterator1 = page.glyphs[Symbol.iterator](), _step1; !(_iteratorNormalCompletion1 = (_step1 = _iterator1.next()).done); _iteratorNormalCompletion1 = true){
|
|
85
|
+
var glyph = _step1.value;
|
|
86
|
+
heightChars.set(glyph.h, ((_heightChars_get = heightChars.get(glyph.h)) !== null && _heightChars_get !== void 0 ? _heightChars_get : 0) + glyph.s.length);
|
|
87
|
+
}
|
|
88
|
+
} catch (err) {
|
|
89
|
+
_didIteratorError1 = true;
|
|
90
|
+
_iteratorError1 = err;
|
|
91
|
+
} finally{
|
|
92
|
+
try {
|
|
93
|
+
if (!_iteratorNormalCompletion1 && _iterator1.return != null) {
|
|
94
|
+
_iterator1.return();
|
|
95
|
+
}
|
|
96
|
+
} finally{
|
|
97
|
+
if (_didIteratorError1) {
|
|
98
|
+
throw _iteratorError1;
|
|
99
|
+
}
|
|
100
|
+
}
|
|
101
|
+
}
|
|
102
|
+
}
|
|
103
|
+
} catch (err) {
|
|
104
|
+
_didIteratorError = true;
|
|
105
|
+
_iteratorError = err;
|
|
106
|
+
} finally{
|
|
107
|
+
try {
|
|
108
|
+
if (!_iteratorNormalCompletion && _iterator.return != null) {
|
|
109
|
+
_iterator.return();
|
|
110
|
+
}
|
|
111
|
+
} finally{
|
|
112
|
+
if (_didIteratorError) {
|
|
113
|
+
throw _iteratorError;
|
|
114
|
+
}
|
|
115
|
+
}
|
|
116
|
+
}
|
|
117
|
+
return {
|
|
118
|
+
numPages: pages.length,
|
|
119
|
+
pages: pages,
|
|
120
|
+
heightChars: heightChars
|
|
121
|
+
};
|
|
122
|
+
}
|
|
123
|
+
function buildTextAnalysis(collected, pageMarkers) {
|
|
124
|
+
var pageLines = collected.pages.map(function(page) {
|
|
125
|
+
return {
|
|
126
|
+
page: page.page,
|
|
127
|
+
lines: (0, _linests.toLines)(page)
|
|
128
|
+
};
|
|
129
|
+
});
|
|
130
|
+
var rendered = (0, _markdownts.buildMarkdown)(pageLines, collected.heightChars, {
|
|
131
|
+
pageMarkers: pageMarkers
|
|
132
|
+
});
|
|
133
|
+
var statsMd = pageMarkers ? rendered.md : (0, _markdownts.buildMarkdown)(pageLines, collected.heightChars, {
|
|
134
|
+
pageMarkers: true
|
|
135
|
+
}).md;
|
|
136
|
+
var hybridPages = collected.pages.filter(function(page) {
|
|
137
|
+
return page.largeImage;
|
|
138
|
+
}).map(function(page) {
|
|
139
|
+
return page.page;
|
|
140
|
+
});
|
|
141
|
+
var report = (0, _reportts.buildReport)({
|
|
142
|
+
numPages: collected.numPages,
|
|
143
|
+
md: rendered.md,
|
|
144
|
+
statsMd: statsMd,
|
|
145
|
+
stats: rendered.stats,
|
|
146
|
+
bodyH: rendered.bodyH,
|
|
147
|
+
headingSizes: rendered.headingSizes,
|
|
148
|
+
hybridPages: hybridPages
|
|
149
|
+
});
|
|
150
|
+
var analysisPages = collected.pages.map(function(page) {
|
|
151
|
+
var _ref;
|
|
152
|
+
var _rendered_pages_find;
|
|
153
|
+
return {
|
|
154
|
+
page: page.page,
|
|
155
|
+
path: 'text',
|
|
156
|
+
words: page.glyphs.map(function(glyph) {
|
|
157
|
+
return sourceWord(glyph, page);
|
|
158
|
+
}),
|
|
159
|
+
blocks: (_ref = (_rendered_pages_find = rendered.pages.find(function(renderedPage) {
|
|
160
|
+
return renderedPage.page === page.page;
|
|
161
|
+
})) === null || _rendered_pages_find === void 0 ? void 0 : _rendered_pages_find.blocks) !== null && _ref !== void 0 ? _ref : []
|
|
162
|
+
};
|
|
163
|
+
});
|
|
164
|
+
var analysis = {
|
|
165
|
+
outputVersion: _reportts.OUTPUT_VERSION,
|
|
166
|
+
report: report,
|
|
167
|
+
pages: analysisPages
|
|
168
|
+
};
|
|
169
|
+
return {
|
|
170
|
+
analysis: analysis,
|
|
171
|
+
markdown: rendered.md,
|
|
172
|
+
words: callbackWords(analysisPages)
|
|
173
|
+
};
|
|
174
|
+
}
|
|
175
|
+
/* CJS INTEROP */ if (exports.__esModule && exports.default) { try { Object.defineProperty(exports.default, '__esModule', { value: true }); for (var key in exports) { exports.default[key] = exports[key]; } } catch (_) {}; module.exports = exports.default; }
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"sources":["/Users/kevin/Dev/OpenSource/formats/pdf-to-md/src/text-analysis.ts"],"sourcesContent":["// Shared text-layer analysis. It keeps PDF source runs separate from layout\n// reconstruction, while Markdown blocks retain the source-run indexes that\n// produced them.\nimport type { PageWord } from './geometry.ts';\nimport { toLines } from './lines.ts';\nimport { buildMarkdown } from './markdown.ts';\nimport { buildReport, OUTPUT_VERSION } from './report.ts';\nimport type { Analysis, AnalysisPage, AnalysisWord, CollectResult, Glyph, PageLines } from './types.ts';\n\nfunction sourceWord(glyph: Glyph, dims: { width: number; height: number }): AnalysisWord {\n return {\n text: glyph.s,\n box: {\n x: dims.width > 0 ? glyph.bounds.x / dims.width : 0,\n y: dims.height > 0 ? (dims.height - glyph.bounds.y - glyph.bounds.h) / dims.height : 0,\n w: dims.width > 0 ? glyph.bounds.w / dims.width : 0,\n h: dims.height > 0 ? glyph.bounds.h / dims.height : 0,\n },\n };\n}\n\nfunction callbackWords(pages: AnalysisPage[]): PageWord[] {\n return pages.flatMap((page) => page.words.map((word) => ({ page: page.page, text: word.text, ...word.box })));\n}\n\nexport interface TextAnalysisResult {\n analysis: Analysis;\n markdown: string;\n words: PageWord[];\n}\n\nexport function selectCollectedPages(collected: CollectResult, pageNumbers: Set<number>): CollectResult {\n const pages = collected.pages.filter((page) => pageNumbers.has(page.page));\n const heightChars = new Map<number, number>();\n for (const page of pages) {\n for (const glyph of page.glyphs) heightChars.set(glyph.h, (heightChars.get(glyph.h) ?? 0) + glyph.s.length);\n }\n return { numPages: pages.length, pages, heightChars };\n}\n\n/** Builds the text result once for Markdown, raw analysis, text callbacks,\n * and reporting. The `pages` returned by buildMarkdown own the actual block\n * construction, so this never reconstructs ownership from rendered text. */\nexport function buildTextAnalysis(collected: CollectResult, pageMarkers: boolean): TextAnalysisResult {\n const pageLines: PageLines[] = collected.pages.map((page) => ({ page: page.page, lines: toLines(page) }));\n const rendered = buildMarkdown(pageLines, collected.heightChars, { pageMarkers });\n const statsMd = pageMarkers ? rendered.md : buildMarkdown(pageLines, collected.heightChars, { pageMarkers: true }).md;\n const hybridPages = collected.pages.filter((page) => page.largeImage).map((page) => page.page);\n const report = buildReport({\n numPages: collected.numPages,\n md: rendered.md,\n statsMd,\n stats: rendered.stats,\n bodyH: rendered.bodyH,\n headingSizes: rendered.headingSizes,\n hybridPages,\n });\n const analysisPages: AnalysisPage[] = collected.pages.map((page) => ({\n page: page.page,\n path: 'text',\n words: page.glyphs.map((glyph) => sourceWord(glyph, page)),\n blocks: rendered.pages.find((renderedPage) => renderedPage.page === page.page)?.blocks ?? [],\n }));\n const analysis: Analysis = { outputVersion: OUTPUT_VERSION, report, pages: analysisPages };\n return { analysis, markdown: rendered.md, words: callbackWords(analysisPages) };\n}\n"],"names":["buildTextAnalysis","selectCollectedPages","sourceWord","glyph","dims","text","s","box","x","width","bounds","y","height","h","w","callbackWords","pages","flatMap","page","words","map","word","collected","pageNumbers","filter","has","heightChars","Map","glyphs","set","get","length","numPages","pageMarkers","pageLines","lines","toLines","rendered","buildMarkdown","statsMd","md","hybridPages","largeImage","report","buildReport","stats","bodyH","headingSizes","analysisPages","path","blocks","find","renderedPage","analysis","outputVersion","OUTPUT_VERSION","markdown"],"mappings":"AAAA,4EAA4E;AAC5E,2EAA2E;AAC3E,iBAAiB;;;;;;;;;;;;QAyCDA;eAAAA;;QAZAC;eAAAA;;;uBA3BQ;0BACM;wBACc;;;;;;;;;;;;;;;;;;;;;;;;;;;AAG5C,SAASC,WAAWC,KAAY,EAAEC,IAAuC;IACvE,OAAO;QACLC,MAAMF,MAAMG,CAAC;QACbC,KAAK;YACHC,GAAGJ,KAAKK,KAAK,GAAG,IAAIN,MAAMO,MAAM,CAACF,CAAC,GAAGJ,KAAKK,KAAK,GAAG;YAClDE,GAAGP,KAAKQ,MAAM,GAAG,IAAI,AAACR,CAAAA,KAAKQ,MAAM,GAAGT,MAAMO,MAAM,CAACC,CAAC,GAAGR,MAAMO,MAAM,CAACG,CAAC,AAADA,IAAKT,KAAKQ,MAAM,GAAG;YACrFE,GAAGV,KAAKK,KAAK,GAAG,IAAIN,MAAMO,MAAM,CAACI,CAAC,GAAGV,KAAKK,KAAK,GAAG;YAClDI,GAAGT,KAAKQ,MAAM,GAAG,IAAIT,MAAMO,MAAM,CAACG,CAAC,GAAGT,KAAKQ,MAAM,GAAG;QACtD;IACF;AACF;AAEA,SAASG,cAAcC,KAAqB;IAC1C,OAAOA,MAAMC,OAAO,CAAC,SAACC;eAASA,KAAKC,KAAK,CAACC,GAAG,CAAC,SAACC;mBAAU;gBAAEH,MAAMA,KAAKA,IAAI;gBAAEb,MAAMgB,KAAKhB,IAAI;eAAKgB,KAAKd,GAAG;;;AAC1G;AAQO,SAASN,qBAAqBqB,SAAwB,EAAEC,WAAwB;IACrF,IAAMP,QAAQM,UAAUN,KAAK,CAACQ,MAAM,CAAC,SAACN;eAASK,YAAYE,GAAG,CAACP,KAAKA,IAAI;;IACxE,IAAMQ,cAAc,IAAIC;QACnB,kCAAA,2BAAA;;QAAL,QAAK,YAAcX,0BAAd,SAAA,6BAAA,QAAA,yBAAA,iCAAqB;YAArB,IAAME,OAAN;gBACwDQ;gBAAtD,mCAAA,4BAAA;;gBAAL,QAAK,aAAeR,KAAKU,MAAM,qBAA1B,UAAA,8BAAA,SAAA,0BAAA;oBAAA,IAAMzB,QAAN;oBAA4BuB,YAAYG,GAAG,CAAC1B,MAAMU,CAAC,EAAE,EAACa,mBAAAA,YAAYI,GAAG,CAAC3B,MAAMU,CAAC,eAAvBa,8BAAAA,mBAA4B,KAAKvB,MAAMG,CAAC,CAACyB,MAAM;;;gBAArG;gBAAA;;;yBAAA,8BAAA;wBAAA;;;wBAAA;8BAAA;;;;QACP;;QAFK;QAAA;;;iBAAA,6BAAA;gBAAA;;;gBAAA;sBAAA;;;;IAGL,OAAO;QAAEC,UAAUhB,MAAMe,MAAM;QAAEf,OAAAA;QAAOU,aAAAA;IAAY;AACtD;AAKO,SAAS1B,kBAAkBsB,SAAwB,EAAEW,WAAoB;IAC9E,IAAMC,YAAyBZ,UAAUN,KAAK,CAACI,GAAG,CAAC,SAACF;eAAU;YAAEA,MAAMA,KAAKA,IAAI;YAAEiB,OAAOC,IAAAA,gBAAO,EAAClB;QAAM;;IACtG,IAAMmB,WAAWC,IAAAA,yBAAa,EAACJ,WAAWZ,UAAUI,WAAW,EAAE;QAAEO,aAAAA;IAAY;IAC/E,IAAMM,UAAUN,cAAcI,SAASG,EAAE,GAAGF,IAAAA,yBAAa,EAACJ,WAAWZ,UAAUI,WAAW,EAAE;QAAEO,aAAa;IAAK,GAAGO,EAAE;IACrH,IAAMC,cAAcnB,UAAUN,KAAK,CAACQ,MAAM,CAAC,SAACN;eAASA,KAAKwB,UAAU;OAAEtB,GAAG,CAAC,SAACF;eAASA,KAAKA,IAAI;;IAC7F,IAAMyB,SAASC,IAAAA,qBAAW,EAAC;QACzBZ,UAAUV,UAAUU,QAAQ;QAC5BQ,IAAIH,SAASG,EAAE;QACfD,SAAAA;QACAM,OAAOR,SAASQ,KAAK;QACrBC,OAAOT,SAASS,KAAK;QACrBC,cAAcV,SAASU,YAAY;QACnCN,aAAAA;IACF;IACA,IAAMO,gBAAgC1B,UAAUN,KAAK,CAACI,GAAG,CAAC,SAACF;;YAIjDmB;eAJ2D;YACnEnB,MAAMA,KAAKA,IAAI;YACf+B,MAAM;YACN9B,OAAOD,KAAKU,MAAM,CAACR,GAAG,CAAC,SAACjB;uBAAUD,WAAWC,OAAOe;;YACpDgC,MAAM,WAAEb,uBAAAA,SAASrB,KAAK,CAACmC,IAAI,CAAC,SAACC;uBAAiBA,aAAalC,IAAI,KAAKA,KAAKA,IAAI;4BAArEmB,2CAAAA,qBAAwEa,MAAM,uCAAI,EAAE;QAC9F;;IACA,IAAMG,WAAqB;QAAEC,eAAeC,wBAAc;QAAEZ,QAAAA;QAAQ3B,OAAOgC;IAAc;IACzF,OAAO;QAAEK,UAAAA;QAAUG,UAAUnB,SAASG,EAAE;QAAErB,OAAOJ,cAAciC;IAAe;AAChF"}
|
package/dist/cjs/types.d.cts
CHANGED
|
@@ -1,9 +1,18 @@
|
|
|
1
|
+
/** Viewport-point AABB for a source text run's transformed advance by
|
|
2
|
+
* font-height box. This is a run box, not fabricated per-token geometry. */
|
|
3
|
+
export interface GlyphBounds {
|
|
4
|
+
x: number;
|
|
5
|
+
y: number;
|
|
6
|
+
w: number;
|
|
7
|
+
h: number;
|
|
8
|
+
}
|
|
1
9
|
export interface Glyph {
|
|
2
10
|
s: string;
|
|
3
11
|
x: number;
|
|
4
12
|
y: number;
|
|
5
13
|
w: number;
|
|
6
14
|
h: number;
|
|
15
|
+
bounds: GlyphBounds;
|
|
7
16
|
}
|
|
8
17
|
export interface CollectedPage {
|
|
9
18
|
page: number;
|
|
@@ -11,6 +20,8 @@ export interface CollectedPage {
|
|
|
11
20
|
width: number;
|
|
12
21
|
height: number;
|
|
13
22
|
textChars: number;
|
|
23
|
+
/** any painted image operator, independent of text-layer density */
|
|
24
|
+
hasImage: boolean;
|
|
14
25
|
largeImage: boolean;
|
|
15
26
|
}
|
|
16
27
|
export interface CollectResult {
|
|
@@ -24,6 +35,9 @@ export interface Line {
|
|
|
24
35
|
y: number;
|
|
25
36
|
h: number;
|
|
26
37
|
col: number;
|
|
38
|
+
/** indexes of the collected PDF text runs that produced this line, in
|
|
39
|
+
* display order. OCR has its own source-word linkage. */
|
|
40
|
+
wordIndexes: number[];
|
|
27
41
|
}
|
|
28
42
|
export interface PageLines {
|
|
29
43
|
page: number;
|
|
@@ -46,11 +60,16 @@ export interface MarkdownResult {
|
|
|
46
60
|
stats: MarkdownStats;
|
|
47
61
|
bodyH: number;
|
|
48
62
|
headingSizes: number[];
|
|
63
|
+
/** The structural blocks that produced `md`, grouped by source page.
|
|
64
|
+
* Markers are rendering provenance and are deliberately not blocks. */
|
|
65
|
+
pages: MarkdownPage[];
|
|
49
66
|
}
|
|
50
67
|
/** One row of the per-page QA breakdown ("### pN" blocks) — used to decide
|
|
51
68
|
* which pages are worth rendering and re-checking (README's "Design contract"). */
|
|
52
69
|
export interface PageStatsRow {
|
|
53
70
|
page: number;
|
|
71
|
+
/** producer retained for this page, including in a mixed result */
|
|
72
|
+
path: 'text' | 'ocr';
|
|
54
73
|
chars: number;
|
|
55
74
|
headings: number;
|
|
56
75
|
junkHeadings: number;
|
|
@@ -92,6 +111,10 @@ export interface WordConfidenceInput {
|
|
|
92
111
|
export interface BuildOcrReportInput {
|
|
93
112
|
md: string;
|
|
94
113
|
fallback?: boolean;
|
|
114
|
+
/** Per-page producer map for a mixed result. Omit for OCR-only output. */
|
|
115
|
+
pagePaths?: Map<number, 'text' | 'ocr'>;
|
|
116
|
+
/** text pages retaining a large embedded image, for mixed per-page QA */
|
|
117
|
+
hybridPages?: number[];
|
|
95
118
|
/** every recognized word, for `report.ts`'s low-confidence detector.
|
|
96
119
|
* Defaults to [] (a pre-confidence --words-json replay), reporting the detector absent, never wrong. */
|
|
97
120
|
words?: WordConfidenceInput[];
|
|
@@ -102,7 +125,7 @@ export interface Report {
|
|
|
102
125
|
/** manually-bumped integer (not the npm package version) — see
|
|
103
126
|
* `report.ts`'s OUTPUT_VERSION for the bump rule; present on every Report so a frozen baseline's staleness is self-announcing. */
|
|
104
127
|
outputVersion: number;
|
|
105
|
-
path: 'text' | 'ocr';
|
|
128
|
+
path: 'text' | 'ocr' | 'mixed';
|
|
106
129
|
pages: number;
|
|
107
130
|
chars: number;
|
|
108
131
|
charsPerPage: number;
|
|
@@ -149,13 +172,20 @@ export interface AnalysisBlock {
|
|
|
149
172
|
/** Heading level (integer 1-6); REQUIRED when `type` is `'heading'`,
|
|
150
173
|
* FORBIDDEN otherwise — toMarkdown()/toText() validate this fail-fast (throws PdfToMdError/'ANALYSIS_INPUT'), never silently ignored. */
|
|
151
174
|
level?: number;
|
|
175
|
+
/** adjacent Markdown line with no blank separator, used for list items */
|
|
176
|
+
tight?: boolean;
|
|
152
177
|
wordIndexes: number[];
|
|
153
178
|
}
|
|
154
179
|
/** One page's flat words plus its structural blocks. */
|
|
155
180
|
export interface AnalysisPage {
|
|
156
181
|
page: number;
|
|
157
|
-
/**
|
|
158
|
-
*
|
|
182
|
+
/** producer retained for this page; `Analysis.report.path` is `'mixed'`
|
|
183
|
+
* when the document has more than one page producer. */
|
|
184
|
+
path: 'text' | 'ocr';
|
|
185
|
+
/** Flat, page-relative source items; `AnalysisBlock.wordIndexes` indexes
|
|
186
|
+
* into it. OCR boxes are normalized [0,1]. Text-run boxes are page
|
|
187
|
+
* fractions and may extend outside that range when the PDF run extends past
|
|
188
|
+
* the viewport. Both use a bottom-left origin with y up. */
|
|
159
189
|
words: AnalysisWord[];
|
|
160
190
|
blocks: AnalysisBlock[];
|
|
161
191
|
}
|
|
@@ -169,3 +199,10 @@ export interface Analysis {
|
|
|
169
199
|
report: Report;
|
|
170
200
|
pages: AnalysisPage[];
|
|
171
201
|
}
|
|
202
|
+
/** One page's blocks from the text Markdown builder. This is internal
|
|
203
|
+
* rendering evidence, but it is declared here because the shared text
|
|
204
|
+
* analysis consumes it without parsing Markdown back into structure. */
|
|
205
|
+
export interface MarkdownPage {
|
|
206
|
+
page: number;
|
|
207
|
+
blocks: AnalysisBlock[];
|
|
208
|
+
}
|
package/dist/cjs/types.d.ts
CHANGED
|
@@ -1,9 +1,18 @@
|
|
|
1
|
+
/** Viewport-point AABB for a source text run's transformed advance by
|
|
2
|
+
* font-height box. This is a run box, not fabricated per-token geometry. */
|
|
3
|
+
export interface GlyphBounds {
|
|
4
|
+
x: number;
|
|
5
|
+
y: number;
|
|
6
|
+
w: number;
|
|
7
|
+
h: number;
|
|
8
|
+
}
|
|
1
9
|
export interface Glyph {
|
|
2
10
|
s: string;
|
|
3
11
|
x: number;
|
|
4
12
|
y: number;
|
|
5
13
|
w: number;
|
|
6
14
|
h: number;
|
|
15
|
+
bounds: GlyphBounds;
|
|
7
16
|
}
|
|
8
17
|
export interface CollectedPage {
|
|
9
18
|
page: number;
|
|
@@ -11,6 +20,8 @@ export interface CollectedPage {
|
|
|
11
20
|
width: number;
|
|
12
21
|
height: number;
|
|
13
22
|
textChars: number;
|
|
23
|
+
/** any painted image operator, independent of text-layer density */
|
|
24
|
+
hasImage: boolean;
|
|
14
25
|
largeImage: boolean;
|
|
15
26
|
}
|
|
16
27
|
export interface CollectResult {
|
|
@@ -24,6 +35,9 @@ export interface Line {
|
|
|
24
35
|
y: number;
|
|
25
36
|
h: number;
|
|
26
37
|
col: number;
|
|
38
|
+
/** indexes of the collected PDF text runs that produced this line, in
|
|
39
|
+
* display order. OCR has its own source-word linkage. */
|
|
40
|
+
wordIndexes: number[];
|
|
27
41
|
}
|
|
28
42
|
export interface PageLines {
|
|
29
43
|
page: number;
|
|
@@ -46,11 +60,16 @@ export interface MarkdownResult {
|
|
|
46
60
|
stats: MarkdownStats;
|
|
47
61
|
bodyH: number;
|
|
48
62
|
headingSizes: number[];
|
|
63
|
+
/** The structural blocks that produced `md`, grouped by source page.
|
|
64
|
+
* Markers are rendering provenance and are deliberately not blocks. */
|
|
65
|
+
pages: MarkdownPage[];
|
|
49
66
|
}
|
|
50
67
|
/** One row of the per-page QA breakdown ("### pN" blocks) — used to decide
|
|
51
68
|
* which pages are worth rendering and re-checking (README's "Design contract"). */
|
|
52
69
|
export interface PageStatsRow {
|
|
53
70
|
page: number;
|
|
71
|
+
/** producer retained for this page, including in a mixed result */
|
|
72
|
+
path: 'text' | 'ocr';
|
|
54
73
|
chars: number;
|
|
55
74
|
headings: number;
|
|
56
75
|
junkHeadings: number;
|
|
@@ -92,6 +111,10 @@ export interface WordConfidenceInput {
|
|
|
92
111
|
export interface BuildOcrReportInput {
|
|
93
112
|
md: string;
|
|
94
113
|
fallback?: boolean;
|
|
114
|
+
/** Per-page producer map for a mixed result. Omit for OCR-only output. */
|
|
115
|
+
pagePaths?: Map<number, 'text' | 'ocr'>;
|
|
116
|
+
/** text pages retaining a large embedded image, for mixed per-page QA */
|
|
117
|
+
hybridPages?: number[];
|
|
95
118
|
/** every recognized word, for `report.ts`'s low-confidence detector.
|
|
96
119
|
* Defaults to [] (a pre-confidence --words-json replay), reporting the detector absent, never wrong. */
|
|
97
120
|
words?: WordConfidenceInput[];
|
|
@@ -102,7 +125,7 @@ export interface Report {
|
|
|
102
125
|
/** manually-bumped integer (not the npm package version) — see
|
|
103
126
|
* `report.ts`'s OUTPUT_VERSION for the bump rule; present on every Report so a frozen baseline's staleness is self-announcing. */
|
|
104
127
|
outputVersion: number;
|
|
105
|
-
path: 'text' | 'ocr';
|
|
128
|
+
path: 'text' | 'ocr' | 'mixed';
|
|
106
129
|
pages: number;
|
|
107
130
|
chars: number;
|
|
108
131
|
charsPerPage: number;
|
|
@@ -149,13 +172,20 @@ export interface AnalysisBlock {
|
|
|
149
172
|
/** Heading level (integer 1-6); REQUIRED when `type` is `'heading'`,
|
|
150
173
|
* FORBIDDEN otherwise — toMarkdown()/toText() validate this fail-fast (throws PdfToMdError/'ANALYSIS_INPUT'), never silently ignored. */
|
|
151
174
|
level?: number;
|
|
175
|
+
/** adjacent Markdown line with no blank separator, used for list items */
|
|
176
|
+
tight?: boolean;
|
|
152
177
|
wordIndexes: number[];
|
|
153
178
|
}
|
|
154
179
|
/** One page's flat words plus its structural blocks. */
|
|
155
180
|
export interface AnalysisPage {
|
|
156
181
|
page: number;
|
|
157
|
-
/**
|
|
158
|
-
*
|
|
182
|
+
/** producer retained for this page; `Analysis.report.path` is `'mixed'`
|
|
183
|
+
* when the document has more than one page producer. */
|
|
184
|
+
path: 'text' | 'ocr';
|
|
185
|
+
/** Flat, page-relative source items; `AnalysisBlock.wordIndexes` indexes
|
|
186
|
+
* into it. OCR boxes are normalized [0,1]. Text-run boxes are page
|
|
187
|
+
* fractions and may extend outside that range when the PDF run extends past
|
|
188
|
+
* the viewport. Both use a bottom-left origin with y up. */
|
|
159
189
|
words: AnalysisWord[];
|
|
160
190
|
blocks: AnalysisBlock[];
|
|
161
191
|
}
|
|
@@ -169,3 +199,10 @@ export interface Analysis {
|
|
|
169
199
|
report: Report;
|
|
170
200
|
pages: AnalysisPage[];
|
|
171
201
|
}
|
|
202
|
+
/** One page's blocks from the text Markdown builder. This is internal
|
|
203
|
+
* rendering evidence, but it is declared here because the shared text
|
|
204
|
+
* analysis consumes it without parsing Markdown back into structure. */
|
|
205
|
+
export interface MarkdownPage {
|
|
206
|
+
page: number;
|
|
207
|
+
blocks: AnalysisBlock[];
|
|
208
|
+
}
|
package/dist/cjs/types.js
CHANGED
|
@@ -1,8 +1,7 @@
|
|
|
1
1
|
// Shared shapes threaded through the collect -> lines -> markdown -> report
|
|
2
2
|
// pipeline; kept in one place since every downstream stage uses at least one.
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
"use strict";
|
|
3
|
+
/** Viewport-point AABB for a source text run's transformed advance by
|
|
4
|
+
* font-height box. This is a run box, not fabricated per-token geometry. */ "use strict";
|
|
6
5
|
Object.defineProperty(exports, "__esModule", {
|
|
7
6
|
value: true
|
|
8
7
|
});
|
package/dist/cjs/types.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"sources":["/Users/kevin/Dev/OpenSource/formats/pdf-to-md/src/types.ts"],"sourcesContent":["// Shared shapes threaded through the collect -> lines -> markdown -> report\n// pipeline; kept in one place since every downstream stage uses at least one.\n\n// A positioned glyph run from the PDF's text layer: `s` may be more than one\n// character
|
|
1
|
+
{"version":3,"sources":["/Users/kevin/Dev/OpenSource/formats/pdf-to-md/src/types.ts"],"sourcesContent":["// Shared shapes threaded through the collect -> lines -> markdown -> report\n// pipeline; kept in one place since every downstream stage uses at least one.\n\n/** Viewport-point AABB for a source text run's transformed advance by\n * font-height box. This is a run box, not fabricated per-token geometry. */\nexport interface GlyphBounds {\n x: number;\n y: number;\n w: number;\n h: number;\n}\n\n// A positioned glyph run from the PDF's text layer: `s` may be more than one\n// character. x/y are the transformed baseline origin in viewport points;\n// w/h are its advance and font height, and bounds is their viewport AABB.\nexport interface Glyph {\n s: string;\n x: number;\n y: number;\n w: number;\n h: number;\n bounds: GlyphBounds;\n}\n\n// One page's worth of collected glyphs plus the geometry collect.ts needs\n// downstream (lines.ts's two-column detection, report.ts's hybrid-page flag).\nexport interface CollectedPage {\n page: number;\n glyphs: Glyph[];\n width: number;\n height: number;\n textChars: number;\n /** any painted image operator, independent of text-layer density */\n hasImage: boolean;\n largeImage: boolean;\n}\n\n// collect.ts's return value: selected pages plus the char-weighted\n// font-height histogram, built ONLY from those pages (matters for a `--pages` re-extraction — see collect.ts).\nexport interface CollectResult {\n numPages: number;\n pages: CollectedPage[];\n heightChars: Map<number, number>;\n}\n\n// A visual line assembled from glyphs (lines.ts's `toLines`): `h` is the\n// char-weighted dominant height (not max); `col` is 0 (left/only) or 1 (right, de-braided page).\nexport interface Line {\n text: string;\n x: number;\n y: number;\n h: number;\n col: number;\n /** indexes of the collected PDF text runs that produced this line, in\n * display order. OCR has its own source-word linkage. */\n wordIndexes: number[];\n}\n\n// One page's lines, as src/extract.ts assembles them for buildMarkdown.\nexport interface PageLines {\n page: number;\n lines: Line[];\n}\n\nexport interface BuildMarkdownOptions {\n pageMarkers?: boolean;\n}\n\n// Running counters src/markdown.ts accumulates while building the output,\n// surfaced in the QA report (src/report.ts's buildReport).\nexport interface MarkdownStats {\n headings: number;\n listItems: number;\n paragraphs: number;\n joinedPairs: number;\n reflowed: number;\n callouts: number;\n closedHyphens: number;\n}\n\nexport interface MarkdownResult {\n md: string;\n stats: MarkdownStats;\n bodyH: number;\n headingSizes: number[];\n /** The structural blocks that produced `md`, grouped by source page.\n * Markers are rendering provenance and are deliberately not blocks. */\n pages: MarkdownPage[];\n}\n\n/** One row of the per-page QA breakdown (\"### pN\" blocks) — used to decide\n * which pages are worth rendering and re-checking (README's \"Design contract\"). */\nexport interface PageStatsRow {\n page: number;\n /** producer retained for this page, including in a mixed result */\n path: 'text' | 'ocr';\n chars: number;\n headings: number;\n junkHeadings: number;\n floats: number;\n paragraphs: number;\n danglingLong: number;\n danglingShort: number;\n /** count of `> [review: ...]` lines — blocks the geometry could not order\n * and labeled instead of guessing (README's \"Design contract\"). */\n reviewBlocks: number;\n /** present only on the text path's rows; omitted (not `false`) on the OCR\n * path, which has no per-page hybrid-image signal. */\n largeImages?: boolean;\n /** count of words below `report.ts`'s LOW_CONFIDENCE_THRESHOLD; omitted (not\n * 0) when the page had no confidence data at all, so \"measured 0\" and \"nothing to measure\" stay distinct. */\n lowConfidenceWords?: number;\n /** the suspect tokens behind `lowConfidenceWords`, worst-confidence first\n * and capped (`report.ts`'s LOW_CONFIDENCE_SAMPLE_CAP) so a chart-heavy page can't bloat the JSON. */\n lowConfidenceSample?: string[];\n}\n\n// buildReport's params (src/report.ts) — everything the text (default) path\n// hands over after running collect -> lines -> markdown.\nexport interface BuildReportInput {\n numPages: number;\n md: string;\n /** identical to `md` but always carries \"### pN\" markers, so pageStats is\n * measured independent of the caller's `--page-markers` choice; same string as `md` when markers were requested. */\n statsMd: string;\n stats: MarkdownStats;\n bodyH: number;\n headingSizes: number[];\n hybridPages?: number[];\n}\n\n/** One recognized word's page/text/confidence — `report.ts`'s low-confidence\n * detector input; a structural subset of OcrWordInput kept here so `types.ts` doesn't import the library-layer module that depends on it. */\nexport interface WordConfidenceInput {\n page: number;\n text: string;\n confidence?: number;\n}\n\nexport interface BuildOcrReportInput {\n md: string;\n fallback?: boolean;\n /** Per-page producer map for a mixed result. Omit for OCR-only output. */\n pagePaths?: Map<number, 'text' | 'ocr'>;\n /** text pages retaining a large embedded image, for mixed per-page QA */\n hybridPages?: number[];\n /** every recognized word, for `report.ts`'s low-confidence detector.\n * Defaults to [] (a pre-confidence --words-json replay), reporting the detector absent, never wrong. */\n words?: WordConfidenceInput[];\n}\n\n/** Per-document and per-page QA metrics returned by every extraction entry\n * point, discriminated by `path`; OCR-only and text-only fields are grouped below. */\nexport interface Report {\n /** manually-bumped integer (not the npm package version) — see\n * `report.ts`'s OUTPUT_VERSION for the bump rule; present on every Report so a frozen baseline's staleness is self-announcing. */\n outputVersion: number;\n path: 'text' | 'ocr' | 'mixed';\n pages: number;\n chars: number;\n charsPerPage: number;\n sparse: boolean;\n headings: number;\n junkHeadings: number;\n paragraphs: number;\n callouts: number;\n danglingLong: number;\n danglingShort: number;\n /** total `> [review: ...]` blocks across all pages (sum of pageStats'\n * reviewBlocks); always 0 on the text path — only the OCR path emits review markers. */\n reviewBlocks: number;\n /** document-level sum of pageStats' `lowConfidenceWords` — see that\n * field for why 0 can mean \"measured 0\" or \"nothing to measure\". */\n lowConfidenceWords: number;\n pageStats: PageStatsRow[];\n\n // --- OCR-only ---\n floats?: number;\n ocrFallback?: boolean;\n\n // --- text-only ---\n bodyFontHeight?: number;\n headingSizes?: number[];\n listItems?: number;\n joinedPairs?: number;\n closedHyphens?: number;\n hybridPages?: number[];\n}\n\n// --- Analysis: the raw-mode IR (analyze() produces it; toMarkdown()/toText()\n// in render-analysis.ts render it — see toMarkdown's doc comment for the fail-fast validation contract) ---\n\n/** One recognized/collected word, page-relative; `box` is always present.\n * See `AnalysisPage.words` for the coordinate contract and per-path granularity difference. */\nexport interface AnalysisWord {\n text: string;\n box: { x: number; y: number; w: number; h: number };\n confidence?: number;\n}\n\n/** One structural block; `wordIndexes` links it to its page's `words` list (blocks -> words only).\n * `text` is bare content for `'heading'` (level separate), else the LITERAL markdown line (decoration included, round-trippable) — a pull-quote is `'paragraph'`, not a 5th type. */\nexport interface AnalysisBlock {\n type: 'heading' | 'paragraph' | 'float' | 'review';\n text: string;\n /** Heading level (integer 1-6); REQUIRED when `type` is `'heading'`,\n * FORBIDDEN otherwise — toMarkdown()/toText() validate this fail-fast (throws PdfToMdError/'ANALYSIS_INPUT'), never silently ignored. */\n level?: number;\n /** adjacent Markdown line with no blank separator, used for list items */\n tight?: boolean;\n wordIndexes: number[];\n}\n\n/** One page's flat words plus its structural blocks. */\nexport interface AnalysisPage {\n page: number;\n /** producer retained for this page; `Analysis.report.path` is `'mixed'`\n * when the document has more than one page producer. */\n path: 'text' | 'ocr';\n /** Flat, page-relative source items; `AnalysisBlock.wordIndexes` indexes\n * into it. OCR boxes are normalized [0,1]. Text-run boxes are page\n * fractions and may extend outside that range when the PDF run extends past\n * the viewport. Both use a bottom-left origin with y up. */\n words: AnalysisWord[];\n blocks: AnalysisBlock[];\n}\n\n/** analyze()'s output: plain, JSON-serializable per-page words/blocks plus\n * the same QA `report` the other entry points return. */\nexport interface Analysis {\n /** manually-bumped integer — see `Report.outputVersion` / `report.ts`'s OUTPUT_VERSION. */\n outputVersion: number;\n /** the same QA report extractText/extractOcr/pdfToMarkdown return, always\n * measured off real markdown (OCR: the shared core's output; TEXT: buildMarkdown's reflowed output, not this coarser IR) — never stale relative to the IR. */\n report: Report;\n pages: AnalysisPage[];\n}\n\n/** One page's blocks from the text Markdown builder. This is internal\n * rendering evidence, but it is declared here because the shared text\n * analysis consumes it without parsing Markdown back into structure. */\nexport interface MarkdownPage {\n page: number;\n blocks: AnalysisBlock[];\n}\n"],"names":[],"mappings":"AAAA,4EAA4E;AAC5E,8EAA8E;AAE9E;0EAC0E"}
|
package/dist/esm/analyze.d.ts
CHANGED
|
@@ -28,7 +28,9 @@ export declare function buildOcrAnalysisPages(input: {
|
|
|
28
28
|
* `Analysis`: plain, JSON-serializable per-page `words`/`blocks` plus the same QA `report` the other entry points return. Render with toMarkdown()/toText(), or hand-edit/splice it (no merge API — see toMarkdown's doc comment).
|
|
29
29
|
*
|
|
30
30
|
* @param input a bare pdf path, or `{ words }` to replay a dump (always analyzed as OCR) — unlike extractOcr, never `{ pdfPath }`
|
|
31
|
-
* @param opts see {@link AnalyzeOptions}; for a
|
|
31
|
+
* @param opts see {@link AnalyzeOptions}; for a PDF path, `path` explicitly
|
|
32
|
+
* selects text (default) or OCR. Automatic per-page routing belongs to
|
|
33
|
+
* pdfToMarkdown and the CLI output formats.
|
|
32
34
|
*/
|
|
33
35
|
export declare function analyze(input: string | {
|
|
34
36
|
words: OcrWordInput[];
|
package/dist/esm/analyze.js
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
// analyze() plus the OCR-path core (buildOcrAnalysisPages) that extractOcr
|
|
2
|
-
// shares with it
|
|
2
|
+
// shares with it. Text analysis uses text-analysis.ts so it captures the same
|
|
3
|
+
// blocks, source runs, and report input as extractText.
|
|
3
4
|
import { collect, pageCount } from './collect.js';
|
|
4
5
|
import { dictAvailable, dictUnavailableWarning } from './dict.js';
|
|
5
6
|
import { emitPage, renderEmitBlock } from './emit.js';
|
|
6
7
|
import { recognizePdf } from './engines/tesseract.js';
|
|
7
8
|
import { buildPageOCR, DEFAULT_TUNING, headingLevels } from './geometry.js';
|
|
8
|
-
import { toLines } from './lines.js';
|
|
9
|
-
import { buildMarkdown } from './markdown.js';
|
|
10
9
|
import { renderPagesToMarkdown } from './render-analysis.js';
|
|
11
|
-
import { buildOcrReport,
|
|
10
|
+
import { buildOcrReport, OUTPUT_VERSION } from './report.js';
|
|
11
|
+
import { buildTextAnalysis } from './text-analysis.js';
|
|
12
12
|
import { validateOcrAnalysisInput, validatePages, validateWordsInput } from './validate.js';
|
|
13
13
|
function toAnalysisWord(w) {
|
|
14
14
|
const box = {
|
|
@@ -81,6 +81,7 @@ function toAnalysisBlock(b, wordIndex) {
|
|
|
81
81
|
});
|
|
82
82
|
words = result.words;
|
|
83
83
|
}
|
|
84
|
+
words = words.filter((word)=>word.page >= first && word.page <= last);
|
|
84
85
|
(_opts_onWords = opts.onWords) === null || _opts_onWords === void 0 ? void 0 : _opts_onWords.call(opts, words);
|
|
85
86
|
// Stable per-page index, assigned in recognition/replay order — this IS
|
|
86
87
|
// the page's flat `words` list order; `wordIndex` recovers it later since geometry.ts never clones a Word (object identity survives row-building).
|
|
@@ -122,6 +123,7 @@ function toAnalysisBlock(b, wordIndex) {
|
|
|
122
123
|
}
|
|
123
124
|
pages.push({
|
|
124
125
|
page: pn,
|
|
126
|
+
path: 'ocr',
|
|
125
127
|
words: ((_byPage_get1 = byPage.get(pn)) !== null && _byPage_get1 !== void 0 ? _byPage_get1 : []).map(toAnalysisWord),
|
|
126
128
|
blocks
|
|
127
129
|
});
|
|
@@ -153,21 +155,9 @@ async function analyzeOcr(input, opts) {
|
|
|
153
155
|
pages
|
|
154
156
|
};
|
|
155
157
|
}
|
|
156
|
-
// TEXT path: deliberately NOT routed through the OCR shared-core pattern —
|
|
157
|
-
// buildMarkdown's reflow/hyphen-rejoin/list rules don't fit the 4-type Analysis block union, so this is an independent, intentionally coarser reconstruction (one word/block per visual LINE) — honest, not byte-identical to extractText (see AnalysisPage, src/types.ts).
|
|
158
|
-
// Converts a text-path Line's page-point geometry (src/lines.ts) into the
|
|
159
|
-
// same normalized box contract the OCR path uses (see AnalysisPage.words) — only the coordinate SYSTEM changes; `box.y` anchors on the line's baseline, `box.w` stays 0.
|
|
160
|
-
function textLineBox(l, dims) {
|
|
161
|
-
const { width, height } = dims;
|
|
162
|
-
return {
|
|
163
|
-
x: width > 0 ? l.x / width : 0,
|
|
164
|
-
y: height > 0 ? (height - l.y) / height : 0,
|
|
165
|
-
w: 0,
|
|
166
|
-
h: height > 0 ? l.h / height : 0
|
|
167
|
-
};
|
|
168
|
-
}
|
|
169
158
|
async function analyzeText(pdfPath, opts) {
|
|
170
159
|
var _opts_pages_last, _opts_pageMarkers;
|
|
160
|
+
var _opts_onWords;
|
|
171
161
|
const warn = (m)=>{
|
|
172
162
|
var _opts_onWarning;
|
|
173
163
|
return (_opts_onWarning = opts.onWarning) === null || _opts_onWarning === void 0 ? void 0 : _opts_onWarning.call(opts, m);
|
|
@@ -177,87 +167,26 @@ async function analyzeText(pdfPath, opts) {
|
|
|
177
167
|
first: opts.pages.first,
|
|
178
168
|
last: (_opts_pages_last = opts.pages.last) !== null && _opts_pages_last !== void 0 ? _opts_pages_last : opts.pages.first
|
|
179
169
|
} : {};
|
|
180
|
-
const
|
|
181
|
-
const
|
|
182
|
-
|
|
183
|
-
{
|
|
184
|
-
width: p.width,
|
|
185
|
-
height: p.height
|
|
186
|
-
}
|
|
187
|
-
]));
|
|
188
|
-
const pageLines = pages.map((p)=>({
|
|
189
|
-
page: p.page,
|
|
190
|
-
lines: toLines(p)
|
|
191
|
-
}));
|
|
192
|
-
const pageMarkers = (_opts_pageMarkers = opts.pageMarkers) !== null && _opts_pageMarkers !== void 0 ? _opts_pageMarkers : false;
|
|
193
|
-
const { md, stats, bodyH, headingSizes } = buildMarkdown(pageLines, heightChars, {
|
|
194
|
-
pageMarkers
|
|
195
|
-
});
|
|
196
|
-
const statsMd = pageMarkers ? md : buildMarkdown(pageLines, heightChars, {
|
|
197
|
-
pageMarkers: true
|
|
198
|
-
}).md;
|
|
199
|
-
const hybridPages = pages.filter((p)=>p.largeImage).map((p)=>p.page);
|
|
170
|
+
const collected = await collect(pdfPath, range);
|
|
171
|
+
const result = buildTextAnalysis(collected, (_opts_pageMarkers = opts.pageMarkers) !== null && _opts_pageMarkers !== void 0 ? _opts_pageMarkers : false);
|
|
172
|
+
const hybridPages = collected.pages.filter((page)=>page.largeImage).map((page)=>page.page);
|
|
200
173
|
for (const n of hybridPages){
|
|
201
174
|
warn(`p${n}: text layer used but page carries large image(s); if it has baked-in text this output won't have it — compare (don't blindly replace) against --pages ${n} --ocr, which can itself introduce errors`);
|
|
202
175
|
}
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
statsMd,
|
|
209
|
-
stats,
|
|
210
|
-
bodyH,
|
|
211
|
-
headingSizes,
|
|
212
|
-
hybridPages
|
|
213
|
-
});
|
|
214
|
-
// headingSizes is offset-independent — reused directly, then re-leveled
|
|
215
|
-
// here with offset 0 regardless of `opts.pageMarkers` (see AnalysisPage).
|
|
216
|
-
const levelOf = new Map(headingSizes.slice(0, 6).map((h, i)=>[
|
|
217
|
-
h,
|
|
218
|
-
Math.min(6, i + 1)
|
|
219
|
-
]));
|
|
220
|
-
const analysisPages = pageLines.map(({ page, lines })=>{
|
|
221
|
-
const dims = pageDims.get(page);
|
|
222
|
-
const words = lines.map((l)=>({
|
|
223
|
-
text: l.text,
|
|
224
|
-
box: textLineBox(l, dims)
|
|
225
|
-
}));
|
|
226
|
-
const blocks = lines.map((l, i)=>{
|
|
227
|
-
const level = levelOf.get(l.h);
|
|
228
|
-
return level !== undefined ? {
|
|
229
|
-
type: 'heading',
|
|
230
|
-
text: l.text,
|
|
231
|
-
level,
|
|
232
|
-
wordIndexes: [
|
|
233
|
-
i
|
|
234
|
-
]
|
|
235
|
-
} : {
|
|
236
|
-
type: 'paragraph',
|
|
237
|
-
text: l.text,
|
|
238
|
-
wordIndexes: [
|
|
239
|
-
i
|
|
240
|
-
]
|
|
241
|
-
};
|
|
242
|
-
});
|
|
243
|
-
return {
|
|
244
|
-
page,
|
|
245
|
-
words,
|
|
246
|
-
blocks
|
|
247
|
-
};
|
|
248
|
-
});
|
|
249
|
-
return {
|
|
250
|
-
outputVersion: OUTPUT_VERSION,
|
|
251
|
-
report,
|
|
252
|
-
pages: analysisPages
|
|
253
|
-
};
|
|
176
|
+
for (const page of collected.pages.filter((page)=>page.textChars === 0 && !page.hasImage)){
|
|
177
|
+
warn(`p${page.page}: no extractable text; the page may be blank or contain only vector content`);
|
|
178
|
+
}
|
|
179
|
+
(_opts_onWords = opts.onWords) === null || _opts_onWords === void 0 ? void 0 : _opts_onWords.call(opts, result.words);
|
|
180
|
+
return result.analysis;
|
|
254
181
|
}
|
|
255
182
|
/**
|
|
256
183
|
* Analyze a PDF (or replay an already-recognized OCR word dump) into
|
|
257
184
|
* `Analysis`: plain, JSON-serializable per-page `words`/`blocks` plus the same QA `report` the other entry points return. Render with toMarkdown()/toText(), or hand-edit/splice it (no merge API — see toMarkdown's doc comment).
|
|
258
185
|
*
|
|
259
186
|
* @param input a bare pdf path, or `{ words }` to replay a dump (always analyzed as OCR) — unlike extractOcr, never `{ pdfPath }`
|
|
260
|
-
* @param opts see {@link AnalyzeOptions}; for a
|
|
187
|
+
* @param opts see {@link AnalyzeOptions}; for a PDF path, `path` explicitly
|
|
188
|
+
* selects text (default) or OCR. Automatic per-page routing belongs to
|
|
189
|
+
* pdfToMarkdown and the CLI output formats.
|
|
261
190
|
*/ export async function analyze(input, opts = {}) {
|
|
262
191
|
validatePages(opts.pages);
|
|
263
192
|
if (typeof input !== 'string') return analyzeOcr({
|