@hraness/kb 0.17.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/DISCLOSURE +43 -0
- package/LICENSE +21 -0
- package/README.md +545 -0
- package/dist/agent-context.js +35 -0
- package/dist/agent-guide-audit.js +18 -0
- package/dist/attachments.js +19 -0
- package/dist/authoring.js +33 -0
- package/dist/benchmark.js +19 -0
- package/dist/capture.js +49 -0
- package/dist/cli.js +3646 -0
- package/dist/clip/acquire.js +57 -0
- package/dist/clip/args.js +21 -0
- package/dist/clip/bounded-byte-buffer.js +9 -0
- package/dist/clip/browser-profiles.js +17 -0
- package/dist/clip/bundle-reader.js +11 -0
- package/dist/clip/cli.js +27 -0
- package/dist/clip/cookies.js +21 -0
- package/dist/clip/defuddle-worker.js +84 -0
- package/dist/clip/doctor.js +32 -0
- package/dist/clip/jobs.js +784 -0
- package/dist/clip/network-proxy.js +9 -0
- package/dist/clip/network.js +28 -0
- package/dist/clip/persist.js +30 -0
- package/dist/clip/refresh.js +11 -0
- package/dist/clip/terminal.js +11 -0
- package/dist/evaluation-builder.js +12238 -0
- package/dist/evaluation-kb.js +26 -0
- package/dist/evaluation.js +35 -0
- package/dist/git.js +39 -0
- package/dist/graph.js +39 -0
- package/dist/index-01jj6rbv.js +1103 -0
- package/dist/index-06c9ctr6.js +738 -0
- package/dist/index-0kavxzqj.js +2480 -0
- package/dist/index-1gwbassd.js +743 -0
- package/dist/index-1n418kb9.js +827 -0
- package/dist/index-1vrd1rmn.js +56 -0
- package/dist/index-1xxnjn0d.js +171 -0
- package/dist/index-3rm7cz6h.js +373 -0
- package/dist/index-3v2z4f0q.js +328 -0
- package/dist/index-48pz4jpc.js +392 -0
- package/dist/index-4j3tt0c3.js +234 -0
- package/dist/index-5n05se68.js +307 -0
- package/dist/index-5vwpzb5a.js +764 -0
- package/dist/index-6g2pv9d2.js +370 -0
- package/dist/index-7fzc32gf.js +712 -0
- package/dist/index-84x0vjjp.js +432 -0
- package/dist/index-adx6khj5.js +488 -0
- package/dist/index-b88v3vtm.js +805 -0
- package/dist/index-cv6fh7z5.js +462 -0
- package/dist/index-cxfrakt7.js +1067 -0
- package/dist/index-d13v9ckt.js +150 -0
- package/dist/index-dycjbhw2.js +1943 -0
- package/dist/index-dyqwejk5.js +531 -0
- package/dist/index-e5fbsywq.js +658 -0
- package/dist/index-f984hw45.js +927 -0
- package/dist/index-gh719d91.js +63 -0
- package/dist/index-hgve9rh2.js +2046 -0
- package/dist/index-hya40gb2.js +474 -0
- package/dist/index-j4zgmzjr.js +152 -0
- package/dist/index-jsmvyyvf.js +1166 -0
- package/dist/index-kh0pwvdd.js +1974 -0
- package/dist/index-mqx4nd6v.js +119 -0
- package/dist/index-mxxxytys.js +1254 -0
- package/dist/index-n5dd7r0v.js +711 -0
- package/dist/index-npg9z1a4.js +431 -0
- package/dist/index-pj501bh1.js +140 -0
- package/dist/index-qry4vhxk.js +1041 -0
- package/dist/index-s2gw5aw9.js +199 -0
- package/dist/index-vvdj6kca.js +70 -0
- package/dist/index-w2zc0vwa.js +571 -0
- package/dist/index-x3fthpsc.js +585 -0
- package/dist/index-zr53sf63.js +44 -0
- package/dist/index-zxdy5pby.js +2728 -0
- package/dist/index-zzhgcwyt.js +966 -0
- package/dist/index.js +453 -0
- package/dist/navigation.js +13 -0
- package/dist/pdf.js +85 -0
- package/dist/percolate.js +24 -0
- package/dist/portfolio.js +77 -0
- package/dist/query.js +29 -0
- package/dist/repository-memory.js +60 -0
- package/dist/sdk.js +36 -0
- package/dist/search-rules.js +34 -0
- package/dist/search.js +22 -0
- package/dist/semantic.js +39 -0
- package/dist/source-inbox.js +17 -0
- package/dist/untrusted-content.js +32 -0
- package/dist/url-intelligence.js +74 -0
- package/dist/workflow.js +23 -0
- package/dist/workflows/decision-context.js +20 -0
- package/dist/workflows/explain-change.js +8 -0
- package/dist/workflows/index.js +28 -0
- package/dist/workflows/plan-radar.js +10 -0
- package/package.json +409 -0
- package/skills/kb/AGENTS.md +19 -0
- package/skills/kb/SKILL.md +82 -0
- package/skills/kb/agents/openai.yaml +4 -0
- package/skills/kb/references/pdf-review.md +51 -0
- package/skills/kb/references/percolate.md +144 -0
- package/skills/kb/references/plan-structure.md +105 -0
- package/skills/kb/references/plan.md +115 -0
- package/skills/kb/references/query.md +231 -0
- package/skills/kb/references/refresh.md +133 -0
- package/skills/kb/references/save-pdf.md +125 -0
- package/skills/kb/references/save-url.md +178 -0
- package/skills/kb/references/url-authentication.md +75 -0
- package/skills/kb/references/url-platforms.md +25 -0
- package/src/agent-context.ts +1241 -0
- package/src/agent-guide-audit.ts +750 -0
- package/src/attachments.ts +723 -0
- package/src/authoring.ts +1725 -0
- package/src/benchmark.ts +362 -0
- package/src/capture.ts +68 -0
- package/src/cli.ts +3660 -0
- package/src/clip/acquire.ts +1273 -0
- package/src/clip/archive-today.ts +402 -0
- package/src/clip/args.ts +423 -0
- package/src/clip/assets.ts +262 -0
- package/src/clip/bounded-byte-buffer.ts +75 -0
- package/src/clip/browser-profiles.ts +372 -0
- package/src/clip/bundle-reader.ts +608 -0
- package/src/clip/capture.ts +1054 -0
- package/src/clip/cli.ts +181 -0
- package/src/clip/cookies.ts +512 -0
- package/src/clip/doctor.ts +1110 -0
- package/src/clip/extract.ts +868 -0
- package/src/clip/jobs.ts +959 -0
- package/src/clip/lib.ts +861 -0
- package/src/clip/media.ts +1474 -0
- package/src/clip/metadata-search-tool/Cargo.lock +2498 -0
- package/src/clip/metadata-search-tool/Cargo.toml +18 -0
- package/src/clip/metadata-search-tool/runner.ts +685 -0
- package/src/clip/metadata-search-tool/src/main.rs +389 -0
- package/src/clip/metadata-search.ts +687 -0
- package/src/clip/network-proxy.ts +627 -0
- package/src/clip/network.ts +950 -0
- package/src/clip/package-root.ts +54 -0
- package/src/clip/persist.ts +1472 -0
- package/src/clip/platforms.ts +1824 -0
- package/src/clip/refresh.ts +197 -0
- package/src/clip/structured.ts +432 -0
- package/src/clip/terminal.ts +197 -0
- package/src/clip/url-intelligence.ts +581 -0
- package/src/clip/url-metadata-backfill.ts +616 -0
- package/src/clip/url-metadata-cli.ts +251 -0
- package/src/clip/url-metadata.ts +1397 -0
- package/src/evaluation-analysis-v2.ts +1440 -0
- package/src/evaluation-builder.ts +2539 -0
- package/src/evaluation-corpus-authoring.ts +2052 -0
- package/src/evaluation-evidence.ts +1147 -0
- package/src/evaluation-execution-v2.ts +426 -0
- package/src/evaluation-implementation.ts +131 -0
- package/src/evaluation-kb-closure.ts +1430 -0
- package/src/evaluation-kb-runner-v2.ts +735 -0
- package/src/evaluation-kb-v2.ts +2567 -0
- package/src/evaluation-kb.ts +1018 -0
- package/src/evaluation-measurement-v2.ts +1462 -0
- package/src/evaluation-packing-v2.ts +452 -0
- package/src/evaluation-redaction.ts +28 -0
- package/src/evaluation-v2.ts +3857 -0
- package/src/evaluation.ts +1220 -0
- package/src/git.ts +1276 -0
- package/src/graph.ts +1487 -0
- package/src/index.ts +20 -0
- package/src/init.ts +123 -0
- package/src/navigation.ts +277 -0
- package/src/note-lock.ts +529 -0
- package/src/pdf/args.ts +202 -0
- package/src/pdf/capture.ts +298 -0
- package/src/pdf/cli.ts +159 -0
- package/src/pdf/extract.ts +303 -0
- package/src/pdf/layout.ts +618 -0
- package/src/pdf/markdown.ts +154 -0
- package/src/pdf/model.ts +234 -0
- package/src/pdf/ocr.ts +190 -0
- package/src/pdf/persist.ts +314 -0
- package/src/pdf/source.ts +135 -0
- package/src/pdf/tools.ts +146 -0
- package/src/pdf.ts +92 -0
- package/src/percolate.ts +877 -0
- package/src/portfolio-audit.ts +574 -0
- package/src/portfolio-identity.ts +162 -0
- package/src/portfolio-registry.ts +473 -0
- package/src/portfolio.ts +765 -0
- package/src/query.ts +646 -0
- package/src/repository-memory.ts +1150 -0
- package/src/sdk.ts +1452 -0
- package/src/search-rules.ts +712 -0
- package/src/search.ts +710 -0
- package/src/semantic-runtime.ts +1139 -0
- package/src/semantic.ts +2480 -0
- package/src/source-inbox.ts +232 -0
- package/src/untrusted-content.ts +401 -0
- package/src/url-intelligence.ts +104 -0
- package/src/vault.ts +636 -0
- package/src/workflow.ts +653 -0
- package/src/workflows/decision-context.ts +81 -0
- package/src/workflows/explain-change.ts +73 -0
- package/src/workflows/index.ts +3 -0
- package/src/workflows/plan-radar.ts +101 -0
|
@@ -0,0 +1,1974 @@
|
|
|
1
|
+
// @bun
|
|
2
|
+
import {
|
|
3
|
+
countWords,
|
|
4
|
+
sniffImage
|
|
5
|
+
} from "./index-f984hw45.js";
|
|
6
|
+
import {
|
|
7
|
+
slugify,
|
|
8
|
+
yamlString
|
|
9
|
+
} from "./index-hgve9rh2.js";
|
|
10
|
+
import {
|
|
11
|
+
safeFetch
|
|
12
|
+
} from "./index-e5fbsywq.js";
|
|
13
|
+
import {
|
|
14
|
+
BoundedByteBuffer
|
|
15
|
+
} from "./index-gh719d91.js";
|
|
16
|
+
import {
|
|
17
|
+
redactSensitiveText,
|
|
18
|
+
sanitizeArtifactUrl
|
|
19
|
+
} from "./index-mxxxytys.js";
|
|
20
|
+
import {
|
|
21
|
+
sanitizeTerminalLine,
|
|
22
|
+
sanitizeTerminalText
|
|
23
|
+
} from "./index-1xxnjn0d.js";
|
|
24
|
+
|
|
25
|
+
// src/pdf/args.ts
|
|
26
|
+
var pdfUsage = `kb pdf \u2014 save a local or public remote PDF as an auditable Markdown bundle
|
|
27
|
+
|
|
28
|
+
Usage:
|
|
29
|
+
kb pdf <file-or-url> [--output <directory>] [--slug <slug>] [--annotations <json>] [--force] [--json]
|
|
30
|
+
kb pdf save <file-or-url> [capture options]
|
|
31
|
+
|
|
32
|
+
Capture options:
|
|
33
|
+
--output <directory> Bundle parent (default: KB_PDF_OUTPUT or kb/articles)
|
|
34
|
+
--slug <slug> Override the title-derived bundle name
|
|
35
|
+
--annotations <json> Optional hash-bound image interpretations from an agent
|
|
36
|
+
--timeout-ms <milliseconds> Per-tool timeout
|
|
37
|
+
--max-pdf-bytes <size> Input limit, for example 512mb
|
|
38
|
+
--max-pages <count> Maximum pages to process
|
|
39
|
+
--max-images <count> Maximum embedded images
|
|
40
|
+
--max-asset-bytes <size> Per-image limit
|
|
41
|
+
--max-total-asset-bytes <size>
|
|
42
|
+
--force Replace only a compatible PDF capture bundle
|
|
43
|
+
--json Emit a machine-readable result
|
|
44
|
+
--quiet Suppress progress and warning lines
|
|
45
|
+
`;
|
|
46
|
+
var valueOptions = new Set([
|
|
47
|
+
"--output",
|
|
48
|
+
"--slug",
|
|
49
|
+
"--annotations",
|
|
50
|
+
"--timeout-ms",
|
|
51
|
+
"--max-pdf-bytes",
|
|
52
|
+
"--max-pages",
|
|
53
|
+
"--max-images",
|
|
54
|
+
"--max-asset-bytes",
|
|
55
|
+
"--max-total-asset-bytes"
|
|
56
|
+
]);
|
|
57
|
+
function optionValue(arguments_, index, name) {
|
|
58
|
+
const value = arguments_[index + 1];
|
|
59
|
+
return value === undefined || value.startsWith("--") ? { ok: false, message: `${name} requires a value` } : value;
|
|
60
|
+
}
|
|
61
|
+
function positiveInteger(value, name, maximum) {
|
|
62
|
+
if (!/^\d+$/u.test(value))
|
|
63
|
+
return `${name} must be a positive integer`;
|
|
64
|
+
const parsed = Number(value);
|
|
65
|
+
return Number.isSafeInteger(parsed) && parsed >= 1 && parsed <= maximum ? parsed : `${name} must be between 1 and ${maximum}`;
|
|
66
|
+
}
|
|
67
|
+
function byteSize(value, name, maximum) {
|
|
68
|
+
const match = /^(\d+)(b|kb|mb|gb)?$/iu.exec(value);
|
|
69
|
+
if (match === null || match[1] === undefined) {
|
|
70
|
+
return `${name} must be an integer byte size such as 500000, 25mb, or 1gb`;
|
|
71
|
+
}
|
|
72
|
+
const amount = Number(match[1]);
|
|
73
|
+
const unit = match[2]?.toLowerCase() ?? "b";
|
|
74
|
+
const multiplier = unit === "gb" ? 1024 ** 3 : unit === "mb" ? 1024 ** 2 : unit === "kb" ? 1024 : 1;
|
|
75
|
+
const parsed = amount * multiplier;
|
|
76
|
+
return Number.isSafeInteger(parsed) && parsed >= 1 && parsed <= maximum ? parsed : `${name} must be between 1 byte and ${maximum} bytes`;
|
|
77
|
+
}
|
|
78
|
+
function parsePdfArguments(rawArguments, environment = {}) {
|
|
79
|
+
if (rawArguments.length === 0 || rawArguments[0] === "help" || rawArguments[0] === "--help" || rawArguments[0] === "-h")
|
|
80
|
+
return { ok: true, value: { command: "help" } };
|
|
81
|
+
let cursor = rawArguments[0] === "save" || rawArguments[0] === "capture" ? 1 : 0;
|
|
82
|
+
const positional = [];
|
|
83
|
+
let outputBase = environment.KB_PDF_OUTPUT ?? "kb/articles";
|
|
84
|
+
let slug;
|
|
85
|
+
let interpretationsPath;
|
|
86
|
+
let force = false;
|
|
87
|
+
let json = false;
|
|
88
|
+
let quiet = false;
|
|
89
|
+
let timeoutMs;
|
|
90
|
+
let maxPdfBytes;
|
|
91
|
+
let maxPages;
|
|
92
|
+
let maxImages;
|
|
93
|
+
let maxAssetBytes;
|
|
94
|
+
let maxTotalAssetBytes;
|
|
95
|
+
for (;cursor < rawArguments.length; cursor += 1) {
|
|
96
|
+
const argument = rawArguments[cursor];
|
|
97
|
+
if (argument === undefined)
|
|
98
|
+
continue;
|
|
99
|
+
if (!argument.startsWith("--")) {
|
|
100
|
+
positional.push(argument);
|
|
101
|
+
continue;
|
|
102
|
+
}
|
|
103
|
+
if (argument === "--force") {
|
|
104
|
+
force = true;
|
|
105
|
+
continue;
|
|
106
|
+
}
|
|
107
|
+
if (argument === "--json") {
|
|
108
|
+
json = true;
|
|
109
|
+
continue;
|
|
110
|
+
}
|
|
111
|
+
if (argument === "--quiet") {
|
|
112
|
+
quiet = true;
|
|
113
|
+
continue;
|
|
114
|
+
}
|
|
115
|
+
if (!valueOptions.has(argument)) {
|
|
116
|
+
return { ok: false, message: `unknown PDF option: ${argument}` };
|
|
117
|
+
}
|
|
118
|
+
const rawValue = optionValue(rawArguments, cursor, argument);
|
|
119
|
+
if (typeof rawValue !== "string")
|
|
120
|
+
return rawValue;
|
|
121
|
+
cursor += 1;
|
|
122
|
+
if (argument === "--output")
|
|
123
|
+
outputBase = rawValue;
|
|
124
|
+
else if (argument === "--slug")
|
|
125
|
+
slug = rawValue;
|
|
126
|
+
else if (argument === "--annotations")
|
|
127
|
+
interpretationsPath = rawValue;
|
|
128
|
+
else if (argument === "--timeout-ms") {
|
|
129
|
+
const parsed = positiveInteger(rawValue, argument, 10 * 60000);
|
|
130
|
+
if (typeof parsed === "string")
|
|
131
|
+
return { ok: false, message: parsed };
|
|
132
|
+
timeoutMs = parsed;
|
|
133
|
+
} else if (argument === "--max-pages") {
|
|
134
|
+
const parsed = positiveInteger(rawValue, argument, 1e4);
|
|
135
|
+
if (typeof parsed === "string")
|
|
136
|
+
return { ok: false, message: parsed };
|
|
137
|
+
maxPages = parsed;
|
|
138
|
+
} else if (argument === "--max-images") {
|
|
139
|
+
const parsed = positiveInteger(rawValue, argument, 1e4);
|
|
140
|
+
if (typeof parsed === "string")
|
|
141
|
+
return { ok: false, message: parsed };
|
|
142
|
+
maxImages = parsed;
|
|
143
|
+
} else {
|
|
144
|
+
const maximum = argument === "--max-asset-bytes" ? 2 * 1024 ** 3 : 8 * 1024 ** 3;
|
|
145
|
+
const parsed = byteSize(rawValue, argument, maximum);
|
|
146
|
+
if (typeof parsed === "string")
|
|
147
|
+
return { ok: false, message: parsed };
|
|
148
|
+
if (argument === "--max-pdf-bytes")
|
|
149
|
+
maxPdfBytes = parsed;
|
|
150
|
+
else if (argument === "--max-asset-bytes")
|
|
151
|
+
maxAssetBytes = parsed;
|
|
152
|
+
else
|
|
153
|
+
maxTotalAssetBytes = parsed;
|
|
154
|
+
}
|
|
155
|
+
}
|
|
156
|
+
const input = positional[0];
|
|
157
|
+
if (input === undefined || positional.length !== 1) {
|
|
158
|
+
return { ok: false, message: "kb pdf requires exactly one PDF path or public URL" };
|
|
159
|
+
}
|
|
160
|
+
if (input.length > 64 * 1024) {
|
|
161
|
+
return { ok: false, message: "PDF input exceeds the 65536 code-unit limit" };
|
|
162
|
+
}
|
|
163
|
+
if (outputBase.trim() === "")
|
|
164
|
+
return { ok: false, message: "--output must not be empty" };
|
|
165
|
+
if (slug !== undefined && slug.trim() === "")
|
|
166
|
+
return { ok: false, message: "--slug must not be empty" };
|
|
167
|
+
return {
|
|
168
|
+
ok: true,
|
|
169
|
+
value: {
|
|
170
|
+
command: "capture",
|
|
171
|
+
input,
|
|
172
|
+
outputBase,
|
|
173
|
+
...slug === undefined ? {} : { slug },
|
|
174
|
+
...interpretationsPath === undefined ? {} : { interpretationsPath },
|
|
175
|
+
force,
|
|
176
|
+
json,
|
|
177
|
+
quiet,
|
|
178
|
+
...timeoutMs === undefined ? {} : { timeoutMs },
|
|
179
|
+
...maxPdfBytes === undefined ? {} : { maxPdfBytes },
|
|
180
|
+
...maxPages === undefined ? {} : { maxPages },
|
|
181
|
+
...maxImages === undefined ? {} : { maxImages },
|
|
182
|
+
...maxAssetBytes === undefined ? {} : { maxAssetBytes },
|
|
183
|
+
...maxTotalAssetBytes === undefined ? {} : { maxTotalAssetBytes }
|
|
184
|
+
}
|
|
185
|
+
};
|
|
186
|
+
}
|
|
187
|
+
|
|
188
|
+
// src/pdf/layout.ts
|
|
189
|
+
var MAX_XML_ATTRIBUTE_CODE_UNITS = 64 * 1024;
|
|
190
|
+
var MAX_XML_TEXT_CODE_UNITS = 2 * 1024 * 1024;
|
|
191
|
+
var isRecord = (value) => typeof value === "object" && value !== null && !Array.isArray(value);
|
|
192
|
+
function bounded(value, maximum) {
|
|
193
|
+
if (value.length <= maximum)
|
|
194
|
+
return value;
|
|
195
|
+
let end = maximum - 1;
|
|
196
|
+
const final = value.charCodeAt(end - 1);
|
|
197
|
+
if (final >= 55296 && final <= 56319)
|
|
198
|
+
end -= 1;
|
|
199
|
+
return `${value.slice(0, Math.max(0, end))}\u2026`;
|
|
200
|
+
}
|
|
201
|
+
function decodeXmlEntity(entity) {
|
|
202
|
+
const named = {
|
|
203
|
+
amp: "&",
|
|
204
|
+
apos: "'",
|
|
205
|
+
gt: ">",
|
|
206
|
+
lt: "<",
|
|
207
|
+
quot: '"'
|
|
208
|
+
};
|
|
209
|
+
const numeric = /^#(?:x([0-9a-f]+)|(\d+))$/iu.exec(entity);
|
|
210
|
+
if (numeric !== null) {
|
|
211
|
+
const value = Number.parseInt(numeric[1] ?? numeric[2] ?? "", numeric[1] === undefined ? 10 : 16);
|
|
212
|
+
if (Number.isSafeInteger(value) && value > 0 && value <= 1114111 && !(value >= 55296 && value <= 57343))
|
|
213
|
+
return String.fromCodePoint(value);
|
|
214
|
+
return "\uFFFD";
|
|
215
|
+
}
|
|
216
|
+
return named[entity] ?? `&${entity};`;
|
|
217
|
+
}
|
|
218
|
+
function decodePopplerText(value) {
|
|
219
|
+
const withoutMarkup = value.replace(/<[^>]{0,65536}>/gu, "");
|
|
220
|
+
const decoded = withoutMarkup.replace(/&([a-z]+|#x[0-9a-f]+|#\d+);/giu, (_match, entity) => decodeXmlEntity(entity.toLowerCase()));
|
|
221
|
+
return sanitizeTerminalText(decoded).replace(/[\uFB01\uFB02\uFB00\uFB03\uFB04]/gu, (ligature) => ({
|
|
222
|
+
"\uFB01": "fi",
|
|
223
|
+
"\uFB02": "fl",
|
|
224
|
+
"\uFB00": "ff",
|
|
225
|
+
"\uFB03": "ffi",
|
|
226
|
+
"\uFB04": "ffl"
|
|
227
|
+
})[ligature] ?? ligature).normalize("NFC");
|
|
228
|
+
}
|
|
229
|
+
function attributes(value) {
|
|
230
|
+
const result = {};
|
|
231
|
+
const pattern = /([a-zA-Z][a-zA-Z0-9_-]*)="([^"]*)"/gu;
|
|
232
|
+
for (const match of value.matchAll(pattern)) {
|
|
233
|
+
const key = match[1];
|
|
234
|
+
const raw = match[2];
|
|
235
|
+
if (key === undefined || raw === undefined || raw.length > MAX_XML_ATTRIBUTE_CODE_UNITS)
|
|
236
|
+
continue;
|
|
237
|
+
result[key] = decodePopplerText(raw);
|
|
238
|
+
}
|
|
239
|
+
return result;
|
|
240
|
+
}
|
|
241
|
+
function nonNegativeNumber(value, label) {
|
|
242
|
+
const parsed = Number(value);
|
|
243
|
+
if (!Number.isFinite(parsed) || parsed < 0 || parsed > 1e8) {
|
|
244
|
+
throw new Error(`Poppler XML contains an invalid ${label}`);
|
|
245
|
+
}
|
|
246
|
+
return parsed;
|
|
247
|
+
}
|
|
248
|
+
function positiveInteger2(value, label) {
|
|
249
|
+
const parsed = Number(value);
|
|
250
|
+
if (!Number.isSafeInteger(parsed) || parsed < 1 || parsed > 1e6) {
|
|
251
|
+
throw new Error(`Poppler XML contains an invalid ${label}`);
|
|
252
|
+
}
|
|
253
|
+
return parsed;
|
|
254
|
+
}
|
|
255
|
+
function parsePopplerXml(xml, limits) {
|
|
256
|
+
const maxTextFragments = Math.max(1, Math.min(limits.maxTextFragments ?? 500000, 2000000));
|
|
257
|
+
const root = /<pdf2xml\b([^>]*)>/u.exec(xml);
|
|
258
|
+
const popplerVersion = root === null ? null : attributes(root[1] ?? "").version ?? null;
|
|
259
|
+
const tokenPattern = /<page\b[^>]*>|<\/page>|<fontspec\b[^>]*\/>|<image\b[^>]*\/>|<text\b[^>]*>[\s\S]*?<\/text>/gu;
|
|
260
|
+
const fonts = new Map;
|
|
261
|
+
const pages = [];
|
|
262
|
+
let current = null;
|
|
263
|
+
let imageCount = 0;
|
|
264
|
+
let textCount = 0;
|
|
265
|
+
let truncated = false;
|
|
266
|
+
for (const tokenMatch of xml.matchAll(tokenPattern)) {
|
|
267
|
+
const token = tokenMatch[0];
|
|
268
|
+
if (token.startsWith("<page")) {
|
|
269
|
+
if (current !== null)
|
|
270
|
+
throw new Error("Poppler XML opened a page before closing the previous page");
|
|
271
|
+
if (pages.length >= limits.maxPages) {
|
|
272
|
+
truncated = true;
|
|
273
|
+
continue;
|
|
274
|
+
}
|
|
275
|
+
const parsed2 = attributes(token);
|
|
276
|
+
current = {
|
|
277
|
+
page: positiveInteger2(parsed2.number, "page number"),
|
|
278
|
+
width: nonNegativeNumber(parsed2.width, "page width"),
|
|
279
|
+
height: nonNegativeNumber(parsed2.height, "page height"),
|
|
280
|
+
text: [],
|
|
281
|
+
images: []
|
|
282
|
+
};
|
|
283
|
+
continue;
|
|
284
|
+
}
|
|
285
|
+
if (token === "</page>") {
|
|
286
|
+
if (current === null)
|
|
287
|
+
continue;
|
|
288
|
+
pages.push(current);
|
|
289
|
+
current = null;
|
|
290
|
+
continue;
|
|
291
|
+
}
|
|
292
|
+
if (current === null)
|
|
293
|
+
continue;
|
|
294
|
+
if (token.startsWith("<fontspec")) {
|
|
295
|
+
const parsed2 = attributes(token);
|
|
296
|
+
const id = parsed2.id;
|
|
297
|
+
if (id !== undefined && id.length <= 256) {
|
|
298
|
+
fonts.set(id, { size: nonNegativeNumber(parsed2.size, "font size") });
|
|
299
|
+
}
|
|
300
|
+
continue;
|
|
301
|
+
}
|
|
302
|
+
if (token.startsWith("<image")) {
|
|
303
|
+
if (imageCount >= limits.maxImages) {
|
|
304
|
+
truncated = true;
|
|
305
|
+
continue;
|
|
306
|
+
}
|
|
307
|
+
const parsed2 = attributes(token);
|
|
308
|
+
const sourcePath = parsed2.src;
|
|
309
|
+
if (sourcePath === undefined || sourcePath === "")
|
|
310
|
+
continue;
|
|
311
|
+
current.images.push({
|
|
312
|
+
page: current.page,
|
|
313
|
+
top: nonNegativeNumber(parsed2.top, "image top"),
|
|
314
|
+
left: nonNegativeNumber(parsed2.left, "image left"),
|
|
315
|
+
width: nonNegativeNumber(parsed2.width, "image width"),
|
|
316
|
+
height: nonNegativeNumber(parsed2.height, "image height"),
|
|
317
|
+
sourcePath
|
|
318
|
+
});
|
|
319
|
+
imageCount += 1;
|
|
320
|
+
continue;
|
|
321
|
+
}
|
|
322
|
+
if (textCount >= maxTextFragments) {
|
|
323
|
+
truncated = true;
|
|
324
|
+
continue;
|
|
325
|
+
}
|
|
326
|
+
const openEnd = token.indexOf(">");
|
|
327
|
+
const closeStart = token.lastIndexOf("</text>");
|
|
328
|
+
if (openEnd < 0 || closeStart <= openEnd)
|
|
329
|
+
continue;
|
|
330
|
+
const parsed = attributes(token.slice(0, openEnd + 1));
|
|
331
|
+
const fontId = parsed.font ?? "";
|
|
332
|
+
const inner = token.slice(openEnd + 1, closeStart);
|
|
333
|
+
if (inner.length > MAX_XML_TEXT_CODE_UNITS) {
|
|
334
|
+
truncated = true;
|
|
335
|
+
continue;
|
|
336
|
+
}
|
|
337
|
+
const text = bounded(decodePopplerText(inner), MAX_XML_TEXT_CODE_UNITS);
|
|
338
|
+
current.text.push({
|
|
339
|
+
top: nonNegativeNumber(parsed.top, "text top"),
|
|
340
|
+
left: nonNegativeNumber(parsed.left, "text left"),
|
|
341
|
+
width: nonNegativeNumber(parsed.width, "text width"),
|
|
342
|
+
height: nonNegativeNumber(parsed.height, "text height"),
|
|
343
|
+
text,
|
|
344
|
+
fontId,
|
|
345
|
+
fontSize: fonts.get(fontId)?.size ?? nonNegativeNumber(parsed.height, "text height"),
|
|
346
|
+
bold: /<b(?:\s[^>]*)?>/iu.test(inner),
|
|
347
|
+
italic: /<i(?:\s[^>]*)?>/iu.test(inner)
|
|
348
|
+
});
|
|
349
|
+
textCount += 1;
|
|
350
|
+
}
|
|
351
|
+
if (current !== null)
|
|
352
|
+
throw new Error("Poppler XML ended before the current page was closed");
|
|
353
|
+
if (pages.length === 0)
|
|
354
|
+
throw new Error("Poppler XML contained no pages");
|
|
355
|
+
return { pages, popplerVersion, truncated };
|
|
356
|
+
}
|
|
357
|
+
function normalizedInfoKey(value) {
|
|
358
|
+
return value.trim().toLowerCase().replace(/\s+/gu, " ");
|
|
359
|
+
}
|
|
360
|
+
function boundedMetadata(value, maximum) {
|
|
361
|
+
const normalized = value?.trim();
|
|
362
|
+
return normalized === undefined || normalized === "" ? null : bounded(sanitizeTerminalText(normalized), maximum);
|
|
363
|
+
}
|
|
364
|
+
function parsePdfInfo(value) {
|
|
365
|
+
const fields = new Map;
|
|
366
|
+
for (const line of value.split(/\r?\n/gu)) {
|
|
367
|
+
const separator = line.indexOf(":");
|
|
368
|
+
if (separator <= 0)
|
|
369
|
+
continue;
|
|
370
|
+
const key = normalizedInfoKey(line.slice(0, separator));
|
|
371
|
+
if (!fields.has(key))
|
|
372
|
+
fields.set(key, line.slice(separator + 1).trim());
|
|
373
|
+
}
|
|
374
|
+
const pageCount = Number(fields.get("pages"));
|
|
375
|
+
if (!Number.isSafeInteger(pageCount) || pageCount < 1 || pageCount > 1e6) {
|
|
376
|
+
throw new Error("pdfinfo did not report a valid positive page count");
|
|
377
|
+
}
|
|
378
|
+
return {
|
|
379
|
+
title: boundedMetadata(fields.get("title"), 2048),
|
|
380
|
+
author: boundedMetadata(fields.get("author"), 1024),
|
|
381
|
+
subject: boundedMetadata(fields.get("subject"), 8192),
|
|
382
|
+
keywords: boundedMetadata(fields.get("keywords"), 8192),
|
|
383
|
+
creator: boundedMetadata(fields.get("creator"), 1024),
|
|
384
|
+
producer: boundedMetadata(fields.get("producer"), 1024),
|
|
385
|
+
createdAt: boundedMetadata(fields.get("creationdate"), 256),
|
|
386
|
+
modifiedAt: boundedMetadata(fields.get("moddate"), 256),
|
|
387
|
+
pageCount,
|
|
388
|
+
encrypted: /^yes\b/iu.test(fields.get("encrypted") ?? "")
|
|
389
|
+
};
|
|
390
|
+
}
|
|
391
|
+
function joinFragments(fragments) {
|
|
392
|
+
const sorted = [...fragments].sort((left, right) => left.left - right.left);
|
|
393
|
+
let output = "";
|
|
394
|
+
let previousRight = 0;
|
|
395
|
+
for (const fragment of sorted) {
|
|
396
|
+
const text = fragment.text;
|
|
397
|
+
if (text === "")
|
|
398
|
+
continue;
|
|
399
|
+
const gap = fragment.left - previousRight;
|
|
400
|
+
const needsSpace = output !== "" && !/\s$/u.test(output) && !/^\s|^[,.;:!?)}\]]/u.test(text) && gap > Math.max(1, fragment.fontSize * 0.08);
|
|
401
|
+
output += `${needsSpace ? " " : ""}${text}`;
|
|
402
|
+
previousRight = Math.max(previousRight, fragment.left + fragment.width);
|
|
403
|
+
}
|
|
404
|
+
return output.replace(/[ \t]+/gu, " ").trim();
|
|
405
|
+
}
|
|
406
|
+
function splitVisualLine(page, pageWidth, fragments) {
|
|
407
|
+
const sorted = [...fragments].sort((left, right) => left.left - right.left);
|
|
408
|
+
const groups = [];
|
|
409
|
+
let current = [];
|
|
410
|
+
let previousRight = 0;
|
|
411
|
+
for (const fragment of sorted) {
|
|
412
|
+
const gap = fragment.left - previousRight;
|
|
413
|
+
if (current.length > 0 && gap > Math.max(pageWidth * 0.2, fragment.fontSize * 8)) {
|
|
414
|
+
groups.push(current);
|
|
415
|
+
current = [];
|
|
416
|
+
}
|
|
417
|
+
current.push(fragment);
|
|
418
|
+
previousRight = Math.max(previousRight, fragment.left + fragment.width);
|
|
419
|
+
}
|
|
420
|
+
if (current.length > 0)
|
|
421
|
+
groups.push(current);
|
|
422
|
+
return groups.flatMap((group) => {
|
|
423
|
+
const text = joinFragments(group);
|
|
424
|
+
if (text === "")
|
|
425
|
+
return [];
|
|
426
|
+
const characters = group.reduce((sum, fragment) => sum + Math.max(1, fragment.text.trim().length), 0);
|
|
427
|
+
const boldCharacters = group.reduce((sum, fragment) => sum + (fragment.bold ? Math.max(1, fragment.text.trim().length) : 0), 0);
|
|
428
|
+
const fontSize = group.reduce((sum, fragment) => sum + fragment.fontSize * Math.max(1, fragment.text.trim().length), 0) / Math.max(1, characters);
|
|
429
|
+
const top = Math.min(...group.map((fragment) => fragment.top));
|
|
430
|
+
const left = Math.min(...group.map((fragment) => fragment.left));
|
|
431
|
+
const right = Math.max(...group.map((fragment) => fragment.left + fragment.width));
|
|
432
|
+
const bottom = Math.max(...group.map((fragment) => fragment.top + fragment.height));
|
|
433
|
+
return [{
|
|
434
|
+
page,
|
|
435
|
+
top,
|
|
436
|
+
left,
|
|
437
|
+
width: right - left,
|
|
438
|
+
height: bottom - top,
|
|
439
|
+
text,
|
|
440
|
+
fontSize,
|
|
441
|
+
boldRatio: boldCharacters / Math.max(1, characters)
|
|
442
|
+
}];
|
|
443
|
+
});
|
|
444
|
+
}
|
|
445
|
+
function pageLines(page) {
|
|
446
|
+
const fragments = [...page.text].sort((left, right) => left.top - right.top || left.left - right.left);
|
|
447
|
+
const rows = [];
|
|
448
|
+
for (const fragment of fragments) {
|
|
449
|
+
const row = rows.at(-1);
|
|
450
|
+
const rowTop = row === undefined ? null : Math.min(...row.map((entry) => entry.top));
|
|
451
|
+
const tolerance = Math.max(2, fragment.height * 0.18);
|
|
452
|
+
if (row === undefined || rowTop === null || Math.abs(fragment.top - rowTop) > tolerance) {
|
|
453
|
+
rows.push([fragment]);
|
|
454
|
+
} else
|
|
455
|
+
row.push(fragment);
|
|
456
|
+
}
|
|
457
|
+
return rows.flatMap((row) => splitVisualLine(page.page, page.width, row)).sort((left, right) => left.top - right.top || left.left - right.left);
|
|
458
|
+
}
|
|
459
|
+
function weightedBodyFontSize(lines) {
|
|
460
|
+
const weights = new Map;
|
|
461
|
+
for (const line of lines) {
|
|
462
|
+
const rounded = Math.round(line.fontSize * 2) / 2;
|
|
463
|
+
weights.set(rounded, (weights.get(rounded) ?? 0) + line.text.length);
|
|
464
|
+
}
|
|
465
|
+
let selected = 12;
|
|
466
|
+
let greatestWeight = -1;
|
|
467
|
+
for (const [size, weight] of weights) {
|
|
468
|
+
if (weight > greatestWeight || weight === greatestWeight && size < selected) {
|
|
469
|
+
selected = size;
|
|
470
|
+
greatestWeight = weight;
|
|
471
|
+
}
|
|
472
|
+
}
|
|
473
|
+
return selected;
|
|
474
|
+
}
|
|
475
|
+
function median(values, fallback) {
|
|
476
|
+
if (values.length === 0)
|
|
477
|
+
return fallback;
|
|
478
|
+
const sorted = [...values].sort((left, right) => left - right);
|
|
479
|
+
return sorted[Math.floor(sorted.length / 2)] ?? fallback;
|
|
480
|
+
}
|
|
481
|
+
function normalizedMarginText(value) {
|
|
482
|
+
return value.toLowerCase().replace(/\d+/gu, "#").replace(/\s+/gu, " ").trim();
|
|
483
|
+
}
|
|
484
|
+
function repeatedMarginLines(pages, linesByPage) {
|
|
485
|
+
const occurrences = new Map;
|
|
486
|
+
for (const page of pages) {
|
|
487
|
+
for (const line of linesByPage.get(page.page) ?? []) {
|
|
488
|
+
const inMargin = line.top <= page.height * 0.06 || line.top + line.height >= page.height * 0.94;
|
|
489
|
+
if (!inMargin || line.text.length > 160)
|
|
490
|
+
continue;
|
|
491
|
+
const normalized = normalizedMarginText(line.text);
|
|
492
|
+
if (normalized === "")
|
|
493
|
+
continue;
|
|
494
|
+
const pageNumbers = occurrences.get(normalized) ?? new Set;
|
|
495
|
+
pageNumbers.add(page.page);
|
|
496
|
+
occurrences.set(normalized, pageNumbers);
|
|
497
|
+
}
|
|
498
|
+
}
|
|
499
|
+
const minimum = Math.max(3, Math.ceil(pages.length * 0.4));
|
|
500
|
+
return new Set([...occurrences.entries()].filter(([, pageNumbers]) => pageNumbers.size >= minimum).map(([text]) => text));
|
|
501
|
+
}
|
|
502
|
+
function headingLevel(fontSize, bodySize, headingSizes) {
|
|
503
|
+
if (fontSize <= bodySize * 1.08)
|
|
504
|
+
return 2;
|
|
505
|
+
const index = headingSizes.findIndex((size) => Math.abs(size - fontSize) < 0.25);
|
|
506
|
+
return Math.min(6, 2 + Math.max(0, index));
|
|
507
|
+
}
|
|
508
|
+
function bulletText(value) {
|
|
509
|
+
const bullet = /^(?:[\u2022\u25CF\u25E6\u25AA\u25AB\u2023\u2043*-]|\d{1,4}[.)]|[a-zA-Z][.)])\s+(.+)$/u.exec(value);
|
|
510
|
+
return bullet?.[1]?.trim() ?? null;
|
|
511
|
+
}
|
|
512
|
+
function layoutBlocks(pages) {
|
|
513
|
+
const linesByPage = new Map(pages.map((page) => [page.page, pageLines(page)]));
|
|
514
|
+
const allLines = [...linesByPage.values()].flat();
|
|
515
|
+
const bodySize = weightedBodyFontSize(allLines);
|
|
516
|
+
const headingSizes = [...new Set(allLines.filter((line) => line.fontSize > bodySize * 1.08).map((line) => Math.round(line.fontSize * 2) / 2))].sort((left, right) => right - left);
|
|
517
|
+
const repeatedMargins = repeatedMarginLines(pages, linesByPage);
|
|
518
|
+
const output = [];
|
|
519
|
+
for (const page of pages) {
|
|
520
|
+
const lines = (linesByPage.get(page.page) ?? []).filter((line) => !repeatedMargins.has(normalizedMarginText(line.text)));
|
|
521
|
+
const steps = lines.slice(1).map((line, index) => Math.max(0, line.top - (lines[index]?.top ?? line.top)));
|
|
522
|
+
const normalStep = median(steps.filter((step) => step > 0), Math.max(1, bodySize * 1.4));
|
|
523
|
+
const events = [
|
|
524
|
+
...lines.map((line, index) => ({ kind: "line", top: line.top, left: line.left, line, index })),
|
|
525
|
+
...page.images.map((image) => ({ kind: "image", top: image.top, left: image.left, image }))
|
|
526
|
+
].sort((left, right) => left.top - right.top || left.left - right.left || (left.kind === "line" ? -1 : 1));
|
|
527
|
+
let paragraph = [];
|
|
528
|
+
const flushParagraph = () => {
|
|
529
|
+
if (paragraph.length === 0)
|
|
530
|
+
return;
|
|
531
|
+
output.push({
|
|
532
|
+
kind: "paragraph",
|
|
533
|
+
page: page.page,
|
|
534
|
+
text: paragraph.join(" ").replace(/\s+/gu, " ").trim()
|
|
535
|
+
});
|
|
536
|
+
paragraph = [];
|
|
537
|
+
};
|
|
538
|
+
for (const event of events) {
|
|
539
|
+
if (event.kind === "image") {
|
|
540
|
+
flushParagraph();
|
|
541
|
+
output.push({ kind: "image", page: page.page, image: event.image });
|
|
542
|
+
continue;
|
|
543
|
+
}
|
|
544
|
+
const { line, index } = event;
|
|
545
|
+
const previous = lines[index - 1];
|
|
546
|
+
const gapBefore = previous === undefined ? Number.POSITIVE_INFINITY : line.top - previous.top;
|
|
547
|
+
const fontHeading = line.fontSize > bodySize * 1.08;
|
|
548
|
+
const boldHeading = line.boldRatio >= 0.78 && line.text.length <= 180 && gapBefore >= normalStep * 1.45;
|
|
549
|
+
if (line.text.length <= 240 && (fontHeading || boldHeading)) {
|
|
550
|
+
flushParagraph();
|
|
551
|
+
output.push({
|
|
552
|
+
kind: "heading",
|
|
553
|
+
page: page.page,
|
|
554
|
+
level: headingLevel(line.fontSize, bodySize, headingSizes),
|
|
555
|
+
text: line.text
|
|
556
|
+
});
|
|
557
|
+
continue;
|
|
558
|
+
}
|
|
559
|
+
const item = bulletText(line.text);
|
|
560
|
+
if (item !== null) {
|
|
561
|
+
flushParagraph();
|
|
562
|
+
output.push({ kind: "list-item", page: page.page, text: item });
|
|
563
|
+
continue;
|
|
564
|
+
}
|
|
565
|
+
if (previous !== undefined && gapBefore >= normalStep * 1.55)
|
|
566
|
+
flushParagraph();
|
|
567
|
+
paragraph.push(line.text);
|
|
568
|
+
}
|
|
569
|
+
flushParagraph();
|
|
570
|
+
}
|
|
571
|
+
return output.filter((block) => block.kind === "image" || block.text.trim() !== "");
|
|
572
|
+
}
|
|
573
|
+
function parsePdfImageInterpretations(value) {
|
|
574
|
+
if (!Array.isArray(value))
|
|
575
|
+
throw new Error("PDF image annotations must be an array");
|
|
576
|
+
if (value.length > 1e4)
|
|
577
|
+
throw new Error("PDF image annotations exceed the 10000-item limit");
|
|
578
|
+
const output = [];
|
|
579
|
+
for (const entry of value) {
|
|
580
|
+
if (!isRecord(entry))
|
|
581
|
+
throw new Error("each PDF image annotation must be an object");
|
|
582
|
+
const id = typeof entry.id === "string" ? entry.id.trim() : "";
|
|
583
|
+
const sha256 = typeof entry.sha256 === "string" ? entry.sha256.trim().toLowerCase() : "";
|
|
584
|
+
if (!/^[a-z0-9][a-z0-9._:-]{0,255}$/u.test(id))
|
|
585
|
+
throw new Error("PDF image annotation has an invalid id");
|
|
586
|
+
if (!/^[0-9a-f]{64}$/u.test(sha256))
|
|
587
|
+
throw new Error("PDF image annotation has an invalid sha256");
|
|
588
|
+
if (entry.kind !== "text" && entry.kind !== "mixed" && entry.kind !== "visual") {
|
|
589
|
+
throw new Error("PDF image annotation kind must be text, mixed, or visual");
|
|
590
|
+
}
|
|
591
|
+
if (entry.method !== undefined && entry.method !== "agent" && entry.method !== "manual") {
|
|
592
|
+
throw new Error("PDF image annotation method must be agent or manual");
|
|
593
|
+
}
|
|
594
|
+
const method = entry.method;
|
|
595
|
+
let metadata;
|
|
596
|
+
if (entry.metadata !== undefined) {
|
|
597
|
+
if (!isRecord(entry.metadata))
|
|
598
|
+
throw new Error("PDF image annotation metadata must be an object");
|
|
599
|
+
const metadataRecord = entry.metadata;
|
|
600
|
+
const optional = (name, maximum = 2048) => {
|
|
601
|
+
const candidate = metadataRecord[name];
|
|
602
|
+
if (candidate === undefined)
|
|
603
|
+
return;
|
|
604
|
+
if (typeof candidate !== "string" || candidate.trim() === "") {
|
|
605
|
+
throw new Error(`PDF image annotation metadata ${name} must be a non-empty string`);
|
|
606
|
+
}
|
|
607
|
+
return bounded(sanitizeTerminalText(candidate.trim()), maximum);
|
|
608
|
+
};
|
|
609
|
+
const participantsValue = metadataRecord.participants;
|
|
610
|
+
let participants;
|
|
611
|
+
if (participantsValue !== undefined) {
|
|
612
|
+
if (!Array.isArray(participantsValue) || participantsValue.length > 256) {
|
|
613
|
+
throw new Error("PDF image annotation participants must be non-empty strings");
|
|
614
|
+
}
|
|
615
|
+
const parsedParticipants = [];
|
|
616
|
+
for (const participant of participantsValue) {
|
|
617
|
+
if (typeof participant !== "string" || participant.trim() === "") {
|
|
618
|
+
throw new Error("PDF image annotation participants must be non-empty strings");
|
|
619
|
+
}
|
|
620
|
+
parsedParticipants.push(bounded(sanitizeTerminalText(participant.trim()), 2048));
|
|
621
|
+
}
|
|
622
|
+
participants = parsedParticipants;
|
|
623
|
+
}
|
|
624
|
+
const platform = optional("platform", 256);
|
|
625
|
+
const contentType = optional("contentType", 256);
|
|
626
|
+
const channel = optional("channel");
|
|
627
|
+
const author = optional("author");
|
|
628
|
+
const timestamp = optional("timestamp", 512);
|
|
629
|
+
metadata = {
|
|
630
|
+
...platform === undefined ? {} : { platform },
|
|
631
|
+
...contentType === undefined ? {} : { contentType },
|
|
632
|
+
...channel === undefined ? {} : { channel },
|
|
633
|
+
...author === undefined ? {} : { author },
|
|
634
|
+
...timestamp === undefined ? {} : { timestamp },
|
|
635
|
+
...participants === undefined ? {} : { participants }
|
|
636
|
+
};
|
|
637
|
+
}
|
|
638
|
+
if (entry.kind === "text" || entry.kind === "mixed") {
|
|
639
|
+
if (typeof entry.markdown !== "string" || entry.markdown.trim() === "") {
|
|
640
|
+
throw new Error("text PDF image annotations require non-empty markdown");
|
|
641
|
+
}
|
|
642
|
+
output.push({
|
|
643
|
+
id,
|
|
644
|
+
sha256,
|
|
645
|
+
kind: entry.kind,
|
|
646
|
+
markdown: bounded(sanitizeTerminalText(entry.markdown.trim()), 2 * 1024 * 1024),
|
|
647
|
+
...metadata === undefined ? {} : { metadata },
|
|
648
|
+
...method === undefined ? {} : { method }
|
|
649
|
+
});
|
|
650
|
+
} else {
|
|
651
|
+
const alt = entry.alt === undefined ? undefined : typeof entry.alt === "string" && entry.alt.trim() !== "" ? bounded(sanitizeTerminalText(entry.alt.trim()), 2048) : (() => {
|
|
652
|
+
throw new Error("visual PDF image annotation alt must be a non-empty string");
|
|
653
|
+
})();
|
|
654
|
+
output.push({
|
|
655
|
+
id,
|
|
656
|
+
sha256,
|
|
657
|
+
kind: "visual",
|
|
658
|
+
...alt === undefined ? {} : { alt },
|
|
659
|
+
...metadata === undefined ? {} : { metadata },
|
|
660
|
+
...method === undefined ? {} : { method }
|
|
661
|
+
});
|
|
662
|
+
}
|
|
663
|
+
}
|
|
664
|
+
const keys = new Set;
|
|
665
|
+
for (const entry of output) {
|
|
666
|
+
if (keys.has(entry.id))
|
|
667
|
+
throw new Error(`duplicate PDF image annotation id: ${entry.id}`);
|
|
668
|
+
keys.add(entry.id);
|
|
669
|
+
}
|
|
670
|
+
return output;
|
|
671
|
+
}
|
|
672
|
+
|
|
673
|
+
// src/pdf/tools.ts
|
|
674
|
+
import { spawn } from "child_process";
|
|
675
|
+
import { existsSync } from "fs";
|
|
676
|
+
import { homedir } from "os";
|
|
677
|
+
import { join } from "path";
|
|
678
|
+
var commonExecutableDirectories = [
|
|
679
|
+
"/opt/homebrew/bin",
|
|
680
|
+
"/usr/local/bin",
|
|
681
|
+
"/usr/bin"
|
|
682
|
+
];
|
|
683
|
+
async function readBoundedStream(stream, maxBytes) {
|
|
684
|
+
const bytes = new BoundedByteBuffer(maxBytes);
|
|
685
|
+
const iterable = stream;
|
|
686
|
+
for await (const value of iterable) {
|
|
687
|
+
let chunk;
|
|
688
|
+
if (typeof value === "string")
|
|
689
|
+
chunk = new TextEncoder().encode(value);
|
|
690
|
+
else if (value instanceof Uint8Array)
|
|
691
|
+
chunk = value;
|
|
692
|
+
else
|
|
693
|
+
throw new Error("PDF tool returned an unsupported output chunk");
|
|
694
|
+
if (!bytes.append(chunk))
|
|
695
|
+
throw new Error(`PDF tool output exceeded ${maxBytes} bytes`);
|
|
696
|
+
}
|
|
697
|
+
return new TextDecoder().decode(bytes.toUint8Array());
|
|
698
|
+
}
|
|
699
|
+
var runPdfToolCommand = async (specification) => {
|
|
700
|
+
const executable = specification.command[0];
|
|
701
|
+
if (executable === undefined)
|
|
702
|
+
throw new Error("PDF tool command is empty");
|
|
703
|
+
const useProcessGroup = process.platform !== "win32";
|
|
704
|
+
const child = spawn(executable, specification.command.slice(1), {
|
|
705
|
+
cwd: specification.cwd,
|
|
706
|
+
detached: useProcessGroup,
|
|
707
|
+
env: {
|
|
708
|
+
...process.env,
|
|
709
|
+
LC_ALL: "C",
|
|
710
|
+
LANG: "C",
|
|
711
|
+
...specification.environment
|
|
712
|
+
},
|
|
713
|
+
stdio: ["ignore", "pipe", "pipe"],
|
|
714
|
+
windowsHide: true
|
|
715
|
+
});
|
|
716
|
+
const exited = new Promise((resolveExit, rejectExit) => {
|
|
717
|
+
child.once("error", rejectExit);
|
|
718
|
+
child.once("close", (code) => resolveExit(code ?? 1));
|
|
719
|
+
});
|
|
720
|
+
const signalProcessTree = (signal) => {
|
|
721
|
+
if (useProcessGroup && child.pid !== undefined) {
|
|
722
|
+
try {
|
|
723
|
+
process.kill(-child.pid, signal);
|
|
724
|
+
return;
|
|
725
|
+
} catch {}
|
|
726
|
+
}
|
|
727
|
+
try {
|
|
728
|
+
child.kill(signal);
|
|
729
|
+
} catch {}
|
|
730
|
+
};
|
|
731
|
+
const state = { failure: null };
|
|
732
|
+
let forceKillTimer = null;
|
|
733
|
+
const requestStop = (error) => {
|
|
734
|
+
state.failure ??= error;
|
|
735
|
+
if (forceKillTimer !== null)
|
|
736
|
+
return;
|
|
737
|
+
signalProcessTree("SIGTERM");
|
|
738
|
+
forceKillTimer = setTimeout(() => signalProcessTree("SIGKILL"), 1000);
|
|
739
|
+
};
|
|
740
|
+
const timer = setTimeout(() => {
|
|
741
|
+
requestStop(new Error(`PDF tool timed out after ${specification.timeoutMs}ms`));
|
|
742
|
+
}, specification.timeoutMs);
|
|
743
|
+
try {
|
|
744
|
+
const [stdout, stderr, exitCode] = await Promise.all([
|
|
745
|
+
readBoundedStream(child.stdout, specification.maxOutputBytes).catch((error) => {
|
|
746
|
+
const normalized = error instanceof Error ? error : new Error(String(error));
|
|
747
|
+
requestStop(normalized);
|
|
748
|
+
throw normalized;
|
|
749
|
+
}),
|
|
750
|
+
readBoundedStream(child.stderr, specification.maxOutputBytes).catch((error) => {
|
|
751
|
+
const normalized = error instanceof Error ? error : new Error(String(error));
|
|
752
|
+
requestStop(normalized);
|
|
753
|
+
throw normalized;
|
|
754
|
+
}),
|
|
755
|
+
exited
|
|
756
|
+
]);
|
|
757
|
+
if (state.failure !== null)
|
|
758
|
+
throw state.failure;
|
|
759
|
+
return { stdout, stderr, exitCode };
|
|
760
|
+
} catch (error) {
|
|
761
|
+
requestStop(error instanceof Error ? error : new Error(String(error)));
|
|
762
|
+
await exited.catch(() => 1);
|
|
763
|
+
throw error;
|
|
764
|
+
} finally {
|
|
765
|
+
clearTimeout(timer);
|
|
766
|
+
if (forceKillTimer !== null)
|
|
767
|
+
clearTimeout(forceKillTimer);
|
|
768
|
+
}
|
|
769
|
+
};
|
|
770
|
+
function discoverExecutable(name, dependencies) {
|
|
771
|
+
const exists = dependencies.exists ?? existsSync;
|
|
772
|
+
const fromPath = (dependencies.which ?? ((value) => Bun.which(value)))(name);
|
|
773
|
+
if (fromPath !== null && exists(fromPath))
|
|
774
|
+
return fromPath;
|
|
775
|
+
const homeCandidates = name === "tesseract" ? [join(homedir(), ".local", "bin", name)] : [];
|
|
776
|
+
for (const path of [
|
|
777
|
+
...homeCandidates,
|
|
778
|
+
...commonExecutableDirectories.map((directory) => join(directory, name))
|
|
779
|
+
]) {
|
|
780
|
+
if (exists(path))
|
|
781
|
+
return path;
|
|
782
|
+
}
|
|
783
|
+
return null;
|
|
784
|
+
}
|
|
785
|
+
function resolvePdfTools(dependencies = {}) {
|
|
786
|
+
const pdfinfo = dependencies.tools?.pdfinfo ?? discoverExecutable("pdfinfo", dependencies);
|
|
787
|
+
const pdftohtml = dependencies.tools?.pdftohtml ?? discoverExecutable("pdftohtml", dependencies);
|
|
788
|
+
const tesseract = dependencies.tools?.tesseract === undefined ? discoverExecutable("tesseract", dependencies) : dependencies.tools.tesseract;
|
|
789
|
+
if (pdfinfo === null) {
|
|
790
|
+
throw new Error("pdfinfo is required for PDF ingestion; install the Poppler command-line tools");
|
|
791
|
+
}
|
|
792
|
+
if (pdftohtml === null) {
|
|
793
|
+
throw new Error("pdftohtml is required for PDF ingestion; install the Poppler command-line tools");
|
|
794
|
+
}
|
|
795
|
+
return { pdfinfo, pdftohtml, tesseract };
|
|
796
|
+
}
|
|
797
|
+
|
|
798
|
+
// src/pdf/extract.ts
|
|
799
|
+
import { createHash } from "crypto";
|
|
800
|
+
import {
|
|
801
|
+
chmodSync,
|
|
802
|
+
closeSync,
|
|
803
|
+
createReadStream,
|
|
804
|
+
existsSync as existsSync2,
|
|
805
|
+
lstatSync,
|
|
806
|
+
mkdirSync,
|
|
807
|
+
openSync,
|
|
808
|
+
readFileSync,
|
|
809
|
+
readSync,
|
|
810
|
+
realpathSync,
|
|
811
|
+
readdirSync,
|
|
812
|
+
statSync
|
|
813
|
+
} from "fs";
|
|
814
|
+
import { basename, isAbsolute, join as join2, relative, resolve, sep } from "path";
|
|
815
|
+
var pdfCaptureDefaults = {
|
|
816
|
+
timeoutMs: 120000,
|
|
817
|
+
maxPdfBytes: 512 * 1024 * 1024,
|
|
818
|
+
maxPages: 500,
|
|
819
|
+
maxImages: 1000,
|
|
820
|
+
maxAssetBytes: 100 * 1024 * 1024,
|
|
821
|
+
maxTotalAssetBytes: 512 * 1024 * 1024,
|
|
822
|
+
maxLayoutBytes: 128 * 1024 * 1024
|
|
823
|
+
};
|
|
824
|
+
function positiveBound(value, fallback, maximum, label) {
|
|
825
|
+
const selected = value ?? fallback;
|
|
826
|
+
if (!Number.isSafeInteger(selected) || selected < 1 || selected > maximum) {
|
|
827
|
+
throw new Error(`${label} must be an integer from 1 through ${maximum}`);
|
|
828
|
+
}
|
|
829
|
+
return selected;
|
|
830
|
+
}
|
|
831
|
+
function pathInside(root, target) {
|
|
832
|
+
const child = relative(root, target);
|
|
833
|
+
return child !== "" && !isAbsolute(child) && child !== ".." && !child.startsWith(`..${sep}`);
|
|
834
|
+
}
|
|
835
|
+
function prepareWorkspace(path) {
|
|
836
|
+
const absolute = resolve(path);
|
|
837
|
+
if (existsSync2(absolute)) {
|
|
838
|
+
const stats = lstatSync(absolute);
|
|
839
|
+
if (!stats.isDirectory() || stats.isSymbolicLink()) {
|
|
840
|
+
throw new Error("PDF inspection workspace must be a real directory");
|
|
841
|
+
}
|
|
842
|
+
if (readdirSync(absolute).length !== 0) {
|
|
843
|
+
throw new Error("PDF inspection workspace must be empty");
|
|
844
|
+
}
|
|
845
|
+
} else {
|
|
846
|
+
mkdirSync(absolute, { recursive: false, mode: 448 });
|
|
847
|
+
}
|
|
848
|
+
chmodSync(absolute, 448);
|
|
849
|
+
return realpathSync(absolute);
|
|
850
|
+
}
|
|
851
|
+
function sourceIdentity(path, maxPdfBytes) {
|
|
852
|
+
const originalFilename = basename(path);
|
|
853
|
+
const canonical = realpathSync(resolve(path));
|
|
854
|
+
const stats = statSync(canonical);
|
|
855
|
+
if (!stats.isFile())
|
|
856
|
+
throw new Error("PDF input must be a regular file");
|
|
857
|
+
if (stats.size < 5)
|
|
858
|
+
throw new Error("PDF input is too small to contain a PDF header");
|
|
859
|
+
if (stats.size > maxPdfBytes)
|
|
860
|
+
throw new Error(`PDF input exceeds the ${maxPdfBytes}-byte limit`);
|
|
861
|
+
const descriptor = openSync(canonical, "r");
|
|
862
|
+
try {
|
|
863
|
+
const signature = Buffer.alloc(5);
|
|
864
|
+
const count = readSync(descriptor, signature, 0, signature.length, 0);
|
|
865
|
+
if (count !== signature.length || signature.toString("ascii") !== "%PDF-") {
|
|
866
|
+
throw new Error("PDF input does not have a valid PDF signature");
|
|
867
|
+
}
|
|
868
|
+
} finally {
|
|
869
|
+
closeSync(descriptor);
|
|
870
|
+
}
|
|
871
|
+
return { inputPath: canonical, originalFilename, bytes: stats.size };
|
|
872
|
+
}
|
|
873
|
+
async function sha256File(path) {
|
|
874
|
+
const digest = createHash("sha256");
|
|
875
|
+
for await (const chunk of createReadStream(path))
|
|
876
|
+
digest.update(chunk);
|
|
877
|
+
return digest.digest("hex");
|
|
878
|
+
}
|
|
879
|
+
function safeGeneratedImage(image, workspace, index, limits) {
|
|
880
|
+
const candidate = resolve(workspace, image.sourcePath);
|
|
881
|
+
if (!pathInside(workspace, candidate)) {
|
|
882
|
+
return { warning: `Skipped an image on page ${image.page} whose generated path escaped the workspace.` };
|
|
883
|
+
}
|
|
884
|
+
let canonical;
|
|
885
|
+
try {
|
|
886
|
+
canonical = realpathSync(candidate);
|
|
887
|
+
} catch {
|
|
888
|
+
return { warning: `Skipped a missing generated image on page ${image.page}.` };
|
|
889
|
+
}
|
|
890
|
+
if (!pathInside(workspace, canonical)) {
|
|
891
|
+
return { warning: `Skipped an image on page ${image.page} whose canonical path escaped the workspace.` };
|
|
892
|
+
}
|
|
893
|
+
const stats = lstatSync(canonical);
|
|
894
|
+
if (!stats.isFile() || stats.isSymbolicLink()) {
|
|
895
|
+
return { warning: `Skipped a non-file generated image on page ${image.page}.` };
|
|
896
|
+
}
|
|
897
|
+
if (stats.size > limits.maxAssetBytes || stats.size > limits.remainingBytes) {
|
|
898
|
+
return { warning: `Skipped an image on page ${image.page} because the configured asset byte limit was reached.` };
|
|
899
|
+
}
|
|
900
|
+
const bytes = readFileSync(canonical);
|
|
901
|
+
const sniffed = sniffImage(bytes);
|
|
902
|
+
if (sniffed === null) {
|
|
903
|
+
return { warning: `Skipped an unsupported generated image on page ${image.page}.` };
|
|
904
|
+
}
|
|
905
|
+
const sha256 = createHash("sha256").update(bytes).digest("hex");
|
|
906
|
+
return {
|
|
907
|
+
id: `page-${image.page}-image-${index + 1}-${sha256.slice(0, 12)}`,
|
|
908
|
+
page: image.page,
|
|
909
|
+
sourcePath: canonical,
|
|
910
|
+
top: image.top,
|
|
911
|
+
left: image.left,
|
|
912
|
+
width: image.width,
|
|
913
|
+
height: image.height,
|
|
914
|
+
bytes: stats.size,
|
|
915
|
+
sha256,
|
|
916
|
+
mimeType: sniffed.mimeType
|
|
917
|
+
};
|
|
918
|
+
}
|
|
919
|
+
function generatedTreeSize(workspace, maximumFiles, maximumBytes) {
|
|
920
|
+
const entries = readdirSync(workspace, { withFileTypes: true });
|
|
921
|
+
if (entries.length > maximumFiles) {
|
|
922
|
+
throw new Error(`PDF extraction created more than ${maximumFiles} files`);
|
|
923
|
+
}
|
|
924
|
+
let totalBytes = 0;
|
|
925
|
+
for (const entry of entries) {
|
|
926
|
+
const path = join2(workspace, entry.name);
|
|
927
|
+
const stats = lstatSync(path);
|
|
928
|
+
if (!entry.isFile() || stats.isSymbolicLink()) {
|
|
929
|
+
throw new Error("PDF extraction created an unexpected non-file output");
|
|
930
|
+
}
|
|
931
|
+
totalBytes += stats.size;
|
|
932
|
+
if (totalBytes > maximumBytes) {
|
|
933
|
+
throw new Error(`PDF extraction exceeded the ${maximumBytes}-byte workspace limit`);
|
|
934
|
+
}
|
|
935
|
+
}
|
|
936
|
+
}
|
|
937
|
+
async function inspectPdf(options, dependencies = {}) {
|
|
938
|
+
const timeoutMs = positiveBound(options.timeoutMs, pdfCaptureDefaults.timeoutMs, 10 * 60000, "timeoutMs");
|
|
939
|
+
const maxPdfBytes = positiveBound(options.maxPdfBytes, pdfCaptureDefaults.maxPdfBytes, 8 * 1024 ** 3, "maxPdfBytes");
|
|
940
|
+
const maxPages = positiveBound(options.maxPages, pdfCaptureDefaults.maxPages, 1e4, "maxPages");
|
|
941
|
+
const maxImages = positiveBound(options.maxImages, pdfCaptureDefaults.maxImages, 1e4, "maxImages");
|
|
942
|
+
const maxAssetBytes = positiveBound(options.maxAssetBytes, pdfCaptureDefaults.maxAssetBytes, 2 * 1024 ** 3, "maxAssetBytes");
|
|
943
|
+
const maxTotalAssetBytes = positiveBound(options.maxTotalAssetBytes, pdfCaptureDefaults.maxTotalAssetBytes, 8 * 1024 ** 3, "maxTotalAssetBytes");
|
|
944
|
+
const source = sourceIdentity(options.inputPath, maxPdfBytes);
|
|
945
|
+
const sourceSha256 = await sha256File(source.inputPath);
|
|
946
|
+
const workspaceDirectory = prepareWorkspace(options.workspaceDirectory);
|
|
947
|
+
const tools = resolvePdfTools(dependencies);
|
|
948
|
+
const runTool = dependencies.runTool ?? runPdfToolCommand;
|
|
949
|
+
const info = await runTool({
|
|
950
|
+
command: [tools.pdfinfo, source.inputPath],
|
|
951
|
+
timeoutMs,
|
|
952
|
+
maxOutputBytes: 2 * 1024 * 1024,
|
|
953
|
+
cwd: workspaceDirectory
|
|
954
|
+
});
|
|
955
|
+
if (info.exitCode !== 0) {
|
|
956
|
+
throw new Error("pdfinfo could not inspect the input PDF");
|
|
957
|
+
}
|
|
958
|
+
const metadata = parsePdfInfo(info.stdout);
|
|
959
|
+
const processedPageLimit = Math.min(metadata.pageCount, maxPages);
|
|
960
|
+
const layoutPath = join2(workspaceDirectory, "layout.xml");
|
|
961
|
+
const extracted = await runTool({
|
|
962
|
+
command: [
|
|
963
|
+
tools.pdftohtml,
|
|
964
|
+
"-q",
|
|
965
|
+
"-f",
|
|
966
|
+
"1",
|
|
967
|
+
"-l",
|
|
968
|
+
String(processedPageLimit),
|
|
969
|
+
"-xml",
|
|
970
|
+
"-hidden",
|
|
971
|
+
"-fmt",
|
|
972
|
+
"png",
|
|
973
|
+
source.inputPath,
|
|
974
|
+
layoutPath
|
|
975
|
+
],
|
|
976
|
+
timeoutMs,
|
|
977
|
+
maxOutputBytes: 2 * 1024 * 1024,
|
|
978
|
+
cwd: workspaceDirectory
|
|
979
|
+
});
|
|
980
|
+
if (extracted.exitCode !== 0) {
|
|
981
|
+
throw new Error("pdftohtml could not extract the input PDF");
|
|
982
|
+
}
|
|
983
|
+
generatedTreeSize(workspaceDirectory, 10002, Math.min(Number.MAX_SAFE_INTEGER, 8 * 1024 ** 3 + pdfCaptureDefaults.maxLayoutBytes));
|
|
984
|
+
const layoutStats = statSync(layoutPath);
|
|
985
|
+
if (!layoutStats.isFile() || layoutStats.size > pdfCaptureDefaults.maxLayoutBytes) {
|
|
986
|
+
throw new Error(`Poppler layout XML exceeds the ${pdfCaptureDefaults.maxLayoutBytes}-byte limit`);
|
|
987
|
+
}
|
|
988
|
+
const parsed = parsePopplerXml(readFileSync(layoutPath, "utf8"), {
|
|
989
|
+
maxPages: processedPageLimit,
|
|
990
|
+
maxImages
|
|
991
|
+
});
|
|
992
|
+
const warnings = [];
|
|
993
|
+
if (metadata.pageCount > processedPageLimit) {
|
|
994
|
+
warnings.push(`PDF extraction stopped at ${processedPageLimit} of ${metadata.pageCount} pages.`);
|
|
995
|
+
}
|
|
996
|
+
if (parsed.truncated) {
|
|
997
|
+
warnings.push("PDF layout extraction reached a configured page, image, or text-fragment limit.");
|
|
998
|
+
}
|
|
999
|
+
if (parsed.pages.length < processedPageLimit) {
|
|
1000
|
+
warnings.push(`Poppler returned ${parsed.pages.length} of ${processedPageLimit} requested pages.`);
|
|
1001
|
+
}
|
|
1002
|
+
let remainingBytes = maxTotalAssetBytes;
|
|
1003
|
+
const pages = parsed.pages.map((page) => {
|
|
1004
|
+
const images = [];
|
|
1005
|
+
const sorted = [...page.images].sort((left, right) => left.top - right.top || left.left - right.left || left.sourcePath.localeCompare(right.sourcePath));
|
|
1006
|
+
for (const [index, rawImage] of sorted.entries()) {
|
|
1007
|
+
const image = safeGeneratedImage(rawImage, workspaceDirectory, index, {
|
|
1008
|
+
maxAssetBytes,
|
|
1009
|
+
remainingBytes
|
|
1010
|
+
});
|
|
1011
|
+
if ("warning" in image) {
|
|
1012
|
+
warnings.push(image.warning);
|
|
1013
|
+
continue;
|
|
1014
|
+
}
|
|
1015
|
+
images.push(image);
|
|
1016
|
+
remainingBytes -= image.bytes;
|
|
1017
|
+
}
|
|
1018
|
+
return {
|
|
1019
|
+
page: page.page,
|
|
1020
|
+
width: page.width,
|
|
1021
|
+
height: page.height,
|
|
1022
|
+
text: page.text,
|
|
1023
|
+
images
|
|
1024
|
+
};
|
|
1025
|
+
});
|
|
1026
|
+
return {
|
|
1027
|
+
inputPath: source.inputPath,
|
|
1028
|
+
originalFilename: source.originalFilename,
|
|
1029
|
+
sourceBytes: source.bytes,
|
|
1030
|
+
sourceSha256,
|
|
1031
|
+
metadata,
|
|
1032
|
+
processedPages: pages.length,
|
|
1033
|
+
pages,
|
|
1034
|
+
popplerVersion: parsed.popplerVersion,
|
|
1035
|
+
warnings: [...new Set(warnings)],
|
|
1036
|
+
workspaceDirectory
|
|
1037
|
+
};
|
|
1038
|
+
}
|
|
1039
|
+
|
|
1040
|
+
// src/pdf/model.ts
|
|
1041
|
+
var PDF_CAPTURE_MANIFEST_SCHEMA_VERSION = 1;
|
|
1042
|
+
var PDF_CAPTURE_MANIFEST_FILENAME = "capture.json";
|
|
1043
|
+
var PDF_CAPTURE_SOURCE_FILENAME = "source.pdf";
|
|
1044
|
+
var PDF_CAPTURE_ANNOTATIONS_FILENAME = "annotations.json";
|
|
1045
|
+
|
|
1046
|
+
// src/pdf/persist.ts
|
|
1047
|
+
import { createHash as createHash2, randomUUID } from "crypto";
|
|
1048
|
+
import {
|
|
1049
|
+
chmodSync as chmodSync2,
|
|
1050
|
+
copyFileSync,
|
|
1051
|
+
existsSync as existsSync3,
|
|
1052
|
+
lstatSync as lstatSync2,
|
|
1053
|
+
mkdirSync as mkdirSync2,
|
|
1054
|
+
mkdtempSync,
|
|
1055
|
+
readFileSync as readFileSync2,
|
|
1056
|
+
readdirSync as readdirSync2,
|
|
1057
|
+
realpathSync as realpathSync2,
|
|
1058
|
+
renameSync,
|
|
1059
|
+
rmSync,
|
|
1060
|
+
statSync as statSync2,
|
|
1061
|
+
writeFileSync
|
|
1062
|
+
} from "fs";
|
|
1063
|
+
import { homedir as homedir2 } from "os";
|
|
1064
|
+
import { dirname, isAbsolute as isAbsolute2, join as join3, relative as relative2, resolve as resolve2, sep as sep2 } from "path";
|
|
1065
|
+
function isConfinedChild(root, path) {
|
|
1066
|
+
const child = relative2(root, path);
|
|
1067
|
+
return child !== "" && !isAbsolute2(child) && child !== ".." && !child.startsWith(`..${sep2}`);
|
|
1068
|
+
}
|
|
1069
|
+
function assertConfinedChild(root, path, label) {
|
|
1070
|
+
if (!isConfinedChild(root, path))
|
|
1071
|
+
throw new Error(`${label} escapes the PDF capture root`);
|
|
1072
|
+
}
|
|
1073
|
+
function safeSlug(slug) {
|
|
1074
|
+
if (slug.length === 0 || slug.length > 240 || [...slug].length > 80 || slug !== slug.normalize("NFKC") || !/^[\p{Letter}\p{Number}](?:[\p{Letter}\p{Number}._-]*[\p{Letter}\p{Number}])?$/u.test(slug))
|
|
1075
|
+
throw new Error("unsafe PDF capture slug");
|
|
1076
|
+
return slug;
|
|
1077
|
+
}
|
|
1078
|
+
function outputRoot(path) {
|
|
1079
|
+
const absolute = resolve2(path);
|
|
1080
|
+
mkdirSync2(absolute, { recursive: true, mode: 493 });
|
|
1081
|
+
const stats = lstatSync2(absolute);
|
|
1082
|
+
if (!stats.isDirectory() && !stats.isSymbolicLink()) {
|
|
1083
|
+
throw new Error("PDF capture output root is not a directory");
|
|
1084
|
+
}
|
|
1085
|
+
const canonical = realpathSync2(absolute);
|
|
1086
|
+
if (!lstatSync2(canonical).isDirectory())
|
|
1087
|
+
throw new Error("PDF capture output root is not a directory");
|
|
1088
|
+
if (dirname(canonical) === canonical || canonical === realpathSync2(homedir2())) {
|
|
1089
|
+
throw new Error("refusing dangerous PDF capture output root");
|
|
1090
|
+
}
|
|
1091
|
+
return canonical;
|
|
1092
|
+
}
|
|
1093
|
+
function extensionForMimeType(mimeType) {
|
|
1094
|
+
switch (mimeType) {
|
|
1095
|
+
case "image/png":
|
|
1096
|
+
return "png";
|
|
1097
|
+
case "image/jpeg":
|
|
1098
|
+
return "jpg";
|
|
1099
|
+
case "image/gif":
|
|
1100
|
+
return "gif";
|
|
1101
|
+
case "image/webp":
|
|
1102
|
+
return "webp";
|
|
1103
|
+
case "image/avif":
|
|
1104
|
+
return "avif";
|
|
1105
|
+
default:
|
|
1106
|
+
throw new Error(`unsupported PDF image MIME type: ${mimeType}`);
|
|
1107
|
+
}
|
|
1108
|
+
}
|
|
1109
|
+
function pdfImageAssetPath(image) {
|
|
1110
|
+
if (!/^[0-9a-f]{64}$/u.test(image.sha256))
|
|
1111
|
+
throw new Error("invalid PDF image sha256");
|
|
1112
|
+
return `assets/${image.sha256}.${extensionForMimeType(image.mimeType)}`;
|
|
1113
|
+
}
|
|
1114
|
+
function hashFile(path) {
|
|
1115
|
+
return createHash2("sha256").update(readFileSync2(path)).digest("hex");
|
|
1116
|
+
}
|
|
1117
|
+
function assertSourceIdentity(path, expectedBytes, expectedSha256, label) {
|
|
1118
|
+
const stats = statSync2(path);
|
|
1119
|
+
if (!stats.isFile() || stats.size !== expectedBytes || hashFile(path) !== expectedSha256) {
|
|
1120
|
+
throw new Error(`${label} changed after PDF inspection`);
|
|
1121
|
+
}
|
|
1122
|
+
}
|
|
1123
|
+
function ownedPdfTarget(targetDirectory, slug) {
|
|
1124
|
+
const directory = lstatSync2(targetDirectory);
|
|
1125
|
+
if (!directory.isDirectory() || directory.isSymbolicLink()) {
|
|
1126
|
+
throw new Error("PDF --force only replaces a regular PDF capture directory");
|
|
1127
|
+
}
|
|
1128
|
+
const manifestPath = join3(targetDirectory, PDF_CAPTURE_MANIFEST_FILENAME);
|
|
1129
|
+
const markdownPath = join3(targetDirectory, `${safeSlug(slug)}.md`);
|
|
1130
|
+
const sourcePath = join3(targetDirectory, PDF_CAPTURE_SOURCE_FILENAME);
|
|
1131
|
+
for (const path of [manifestPath, markdownPath, sourcePath]) {
|
|
1132
|
+
const stats = lstatSync2(path);
|
|
1133
|
+
if (!stats.isFile() || stats.isSymbolicLink()) {
|
|
1134
|
+
throw new Error("PDF --force refused a target without its owned files");
|
|
1135
|
+
}
|
|
1136
|
+
}
|
|
1137
|
+
if (lstatSync2(manifestPath).size > 16 * 1024 * 1024) {
|
|
1138
|
+
throw new Error("PDF --force refused an oversized capture manifest");
|
|
1139
|
+
}
|
|
1140
|
+
let parsed;
|
|
1141
|
+
try {
|
|
1142
|
+
parsed = JSON.parse(readFileSync2(manifestPath, "utf8"));
|
|
1143
|
+
} catch {
|
|
1144
|
+
throw new Error("PDF --force refused an invalid capture manifest");
|
|
1145
|
+
}
|
|
1146
|
+
if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed) || !("schemaVersion" in parsed) || parsed.schemaVersion !== PDF_CAPTURE_MANIFEST_SCHEMA_VERSION || !("kind" in parsed) || parsed.kind !== "pdf")
|
|
1147
|
+
throw new Error("PDF --force refused an incompatible capture manifest");
|
|
1148
|
+
return { device: directory.dev, inode: directory.ino };
|
|
1149
|
+
}
|
|
1150
|
+
function assertStagingTreeSafe(root, directory = root) {
|
|
1151
|
+
for (const entry of readdirSync2(directory, { withFileTypes: true })) {
|
|
1152
|
+
const path = join3(directory, entry.name);
|
|
1153
|
+
assertConfinedChild(root, path, "staged PDF artifact");
|
|
1154
|
+
const stats = lstatSync2(path);
|
|
1155
|
+
if (stats.isSymbolicLink())
|
|
1156
|
+
throw new Error("staged PDF artifacts must not be symbolic links");
|
|
1157
|
+
if (stats.isDirectory())
|
|
1158
|
+
assertStagingTreeSafe(root, path);
|
|
1159
|
+
else if (!stats.isFile())
|
|
1160
|
+
throw new Error("staged PDF artifacts must be regular files");
|
|
1161
|
+
}
|
|
1162
|
+
}
|
|
1163
|
+
function unusedBackupPath(root) {
|
|
1164
|
+
for (;; ) {
|
|
1165
|
+
const candidate = join3(root, `.pdf-capture-backup-${randomUUID()}`);
|
|
1166
|
+
assertConfinedChild(root, candidate, "PDF capture backup");
|
|
1167
|
+
if (!existsSync3(candidate))
|
|
1168
|
+
return candidate;
|
|
1169
|
+
}
|
|
1170
|
+
}
|
|
1171
|
+
function errorMessage(error) {
|
|
1172
|
+
return error instanceof Error ? error.message : String(error);
|
|
1173
|
+
}
|
|
1174
|
+
function persistPdfCapture(input) {
|
|
1175
|
+
const slug = safeSlug(input.slug);
|
|
1176
|
+
const root = outputRoot(input.outputBase);
|
|
1177
|
+
const targetDirectory = join3(root, slug);
|
|
1178
|
+
assertConfinedChild(root, targetDirectory, "PDF capture target");
|
|
1179
|
+
const targetExists = existsSync3(targetDirectory);
|
|
1180
|
+
if (targetExists && !input.force) {
|
|
1181
|
+
throw new Error(`PDF capture already exists: ${targetDirectory}; pass --force to replace it`);
|
|
1182
|
+
}
|
|
1183
|
+
const expectedIdentity = targetExists ? ownedPdfTarget(targetDirectory, slug) : null;
|
|
1184
|
+
const stagingDirectory = mkdtempSync(join3(root, ".pdf-capture-staging-"));
|
|
1185
|
+
chmodSync2(stagingDirectory, 448);
|
|
1186
|
+
assertConfinedChild(root, stagingDirectory, "PDF capture staging directory");
|
|
1187
|
+
try {
|
|
1188
|
+
assertSourceIdentity(input.sourcePath, input.manifest.source.bytes, input.manifest.source.sha256, "PDF source");
|
|
1189
|
+
const sourceDestination = join3(stagingDirectory, PDF_CAPTURE_SOURCE_FILENAME);
|
|
1190
|
+
copyFileSync(input.sourcePath, sourceDestination);
|
|
1191
|
+
chmodSync2(sourceDestination, 420);
|
|
1192
|
+
const assetsDirectory = join3(stagingDirectory, "assets");
|
|
1193
|
+
mkdirSync2(assetsDirectory, { recursive: true, mode: 493 });
|
|
1194
|
+
const imagesByPath = new Map;
|
|
1195
|
+
for (const image of input.images) {
|
|
1196
|
+
const assetPath = pdfImageAssetPath(image);
|
|
1197
|
+
const existing = imagesByPath.get(assetPath);
|
|
1198
|
+
if (existing !== undefined) {
|
|
1199
|
+
if (existing.bytes !== image.bytes || existing.sha256 !== image.sha256) {
|
|
1200
|
+
throw new Error("conflicting PDF image assets have the same destination");
|
|
1201
|
+
}
|
|
1202
|
+
continue;
|
|
1203
|
+
}
|
|
1204
|
+
imagesByPath.set(assetPath, image);
|
|
1205
|
+
}
|
|
1206
|
+
for (const [assetPath, image] of [...imagesByPath.entries()].sort(([left], [right]) => left.localeCompare(right))) {
|
|
1207
|
+
assertSourceIdentity(image.sourcePath, image.bytes, image.sha256, "PDF image");
|
|
1208
|
+
const destination = join3(stagingDirectory, assetPath);
|
|
1209
|
+
assertConfinedChild(stagingDirectory, destination, "PDF image destination");
|
|
1210
|
+
copyFileSync(image.sourcePath, destination);
|
|
1211
|
+
chmodSync2(destination, 420);
|
|
1212
|
+
}
|
|
1213
|
+
writeFileSync(join3(stagingDirectory, `${slug}.md`), `${redactSensitiveText(input.markdown).trimEnd()}
|
|
1214
|
+
`, { encoding: "utf8", flag: "wx", mode: 420 });
|
|
1215
|
+
if (input.manifest.annotations === null) {
|
|
1216
|
+
if (input.annotationsJson !== undefined) {
|
|
1217
|
+
throw new Error("PDF annotations were supplied without manifest provenance");
|
|
1218
|
+
}
|
|
1219
|
+
} else {
|
|
1220
|
+
const annotationsJson = input.annotationsJson;
|
|
1221
|
+
if (annotationsJson === undefined) {
|
|
1222
|
+
throw new Error("PDF annotation provenance is missing its retained input");
|
|
1223
|
+
}
|
|
1224
|
+
if (Buffer.byteLength(annotationsJson) !== input.manifest.annotations.bytes || createHash2("sha256").update(annotationsJson).digest("hex") !== input.manifest.annotations.sha256) {
|
|
1225
|
+
throw new Error("PDF annotation input does not match its manifest provenance");
|
|
1226
|
+
}
|
|
1227
|
+
let annotations;
|
|
1228
|
+
try {
|
|
1229
|
+
annotations = JSON.parse(annotationsJson);
|
|
1230
|
+
} catch {
|
|
1231
|
+
throw new Error("PDF annotation input is not valid JSON");
|
|
1232
|
+
}
|
|
1233
|
+
if (!Array.isArray(annotations) || annotations.length !== input.manifest.annotations.count) {
|
|
1234
|
+
throw new Error("PDF annotation input count does not match its manifest provenance");
|
|
1235
|
+
}
|
|
1236
|
+
writeFileSync(join3(stagingDirectory, PDF_CAPTURE_ANNOTATIONS_FILENAME), annotationsJson, { encoding: "utf8", flag: "wx", mode: 420 });
|
|
1237
|
+
}
|
|
1238
|
+
writeFileSync(join3(stagingDirectory, PDF_CAPTURE_MANIFEST_FILENAME), `${JSON.stringify(input.manifest, null, 2)}
|
|
1239
|
+
`, { encoding: "utf8", flag: "wx", mode: 420 });
|
|
1240
|
+
assertStagingTreeSafe(stagingDirectory);
|
|
1241
|
+
if (existsSync3(targetDirectory) !== (expectedIdentity !== null)) {
|
|
1242
|
+
throw new Error("PDF capture target changed during the transaction");
|
|
1243
|
+
}
|
|
1244
|
+
if (expectedIdentity !== null) {
|
|
1245
|
+
const current = ownedPdfTarget(targetDirectory, slug);
|
|
1246
|
+
if (current.device !== expectedIdentity.device || current.inode !== expectedIdentity.inode) {
|
|
1247
|
+
throw new Error("PDF capture target changed during the transaction");
|
|
1248
|
+
}
|
|
1249
|
+
}
|
|
1250
|
+
const backupDirectory = expectedIdentity === null ? null : unusedBackupPath(root);
|
|
1251
|
+
let installed = false;
|
|
1252
|
+
let backedUp = false;
|
|
1253
|
+
try {
|
|
1254
|
+
if (backupDirectory !== null) {
|
|
1255
|
+
renameSync(targetDirectory, backupDirectory);
|
|
1256
|
+
backedUp = true;
|
|
1257
|
+
input.afterBackup?.();
|
|
1258
|
+
}
|
|
1259
|
+
renameSync(stagingDirectory, targetDirectory);
|
|
1260
|
+
installed = true;
|
|
1261
|
+
input.afterInstall?.();
|
|
1262
|
+
if (backupDirectory !== null)
|
|
1263
|
+
rmSync(backupDirectory, { recursive: true, force: true });
|
|
1264
|
+
return targetDirectory;
|
|
1265
|
+
} catch (error) {
|
|
1266
|
+
let rollbackError;
|
|
1267
|
+
try {
|
|
1268
|
+
if (installed && existsSync3(targetDirectory)) {
|
|
1269
|
+
rmSync(targetDirectory, { recursive: true, force: true });
|
|
1270
|
+
}
|
|
1271
|
+
if (backedUp && backupDirectory !== null && existsSync3(backupDirectory)) {
|
|
1272
|
+
if (existsSync3(targetDirectory))
|
|
1273
|
+
throw new Error("target was recreated before rollback");
|
|
1274
|
+
renameSync(backupDirectory, targetDirectory);
|
|
1275
|
+
}
|
|
1276
|
+
} catch (caught) {
|
|
1277
|
+
rollbackError = caught;
|
|
1278
|
+
}
|
|
1279
|
+
if (rollbackError !== undefined) {
|
|
1280
|
+
throw new Error(`PDF capture commit failed (${errorMessage(error)}) and rollback failed (${errorMessage(rollbackError)})`, { cause: error });
|
|
1281
|
+
}
|
|
1282
|
+
throw error;
|
|
1283
|
+
}
|
|
1284
|
+
} catch (error) {
|
|
1285
|
+
if (existsSync3(stagingDirectory))
|
|
1286
|
+
rmSync(stagingDirectory, { recursive: true, force: true });
|
|
1287
|
+
throw error;
|
|
1288
|
+
}
|
|
1289
|
+
}
|
|
1290
|
+
function pdfMarkdownFilename(slug) {
|
|
1291
|
+
return `${safeSlug(slug)}.md`;
|
|
1292
|
+
}
|
|
1293
|
+
|
|
1294
|
+
// src/pdf/markdown.ts
|
|
1295
|
+
function escapeInline(value) {
|
|
1296
|
+
return sanitizeTerminalText(value).replace(/\\/gu, "\\\\").replace(/([`*_[\]{}<>#])/gu, "\\$1").replace(/\s+/gu, " ").trim();
|
|
1297
|
+
}
|
|
1298
|
+
function metadataLine(metadata) {
|
|
1299
|
+
if (metadata === null)
|
|
1300
|
+
return null;
|
|
1301
|
+
const values = [
|
|
1302
|
+
metadata.platform === undefined ? null : `platform: ${metadata.platform}`,
|
|
1303
|
+
metadata.contentType === undefined ? null : `type: ${metadata.contentType}`,
|
|
1304
|
+
metadata.channel === undefined ? null : `channel: ${metadata.channel}`,
|
|
1305
|
+
metadata.author === undefined ? null : `author: ${metadata.author}`,
|
|
1306
|
+
metadata.timestamp === undefined ? null : `timestamp: ${metadata.timestamp}`,
|
|
1307
|
+
metadata.participants === undefined ? null : `participants: ${metadata.participants.join(", ")}`
|
|
1308
|
+
].filter((value) => value !== null);
|
|
1309
|
+
return values.length === 0 ? null : `*Image metadata \u2014 ${escapeInline(values.join("; "))}*`;
|
|
1310
|
+
}
|
|
1311
|
+
function imageMarkdown(image) {
|
|
1312
|
+
const assetPath = pdfImageAssetPath(image.image);
|
|
1313
|
+
const semantic = image.metadata?.contentType ?? image.metadata?.platform;
|
|
1314
|
+
const fallbackAlt = semantic === undefined ? `PDF image from page ${image.image.page}` : `${semantic} from page ${image.image.page}`;
|
|
1315
|
+
const alt = escapeInline(image.alt ?? fallbackAlt);
|
|
1316
|
+
const lines = [``];
|
|
1317
|
+
const metadata = metadataLine(image.metadata);
|
|
1318
|
+
if (metadata !== null)
|
|
1319
|
+
lines.push("", metadata);
|
|
1320
|
+
if (image.kind === "text" || image.kind === "mixed") {
|
|
1321
|
+
const heading = image.metadata?.platform === undefined ? "Text visible in image" : `Text visible in ${escapeInline(image.metadata.platform)} image`;
|
|
1322
|
+
lines.push("", `#### ${heading}`, "", image.markdown.trim());
|
|
1323
|
+
}
|
|
1324
|
+
return lines.join(`
|
|
1325
|
+
`);
|
|
1326
|
+
}
|
|
1327
|
+
function markdownContent(blocks, images) {
|
|
1328
|
+
const lines = [];
|
|
1329
|
+
let currentPage = null;
|
|
1330
|
+
let headingCount = 0;
|
|
1331
|
+
let textBlockCount = 0;
|
|
1332
|
+
for (const block of blocks) {
|
|
1333
|
+
if (currentPage !== block.page) {
|
|
1334
|
+
if (lines.length > 0)
|
|
1335
|
+
lines.push("");
|
|
1336
|
+
lines.push(`<!-- pdf-page: ${block.page} -->`, "");
|
|
1337
|
+
currentPage = block.page;
|
|
1338
|
+
}
|
|
1339
|
+
if (block.kind === "image") {
|
|
1340
|
+
const resolved = images.get(block.image.id);
|
|
1341
|
+
if (resolved === undefined)
|
|
1342
|
+
continue;
|
|
1343
|
+
lines.push(imageMarkdown(resolved), "");
|
|
1344
|
+
continue;
|
|
1345
|
+
}
|
|
1346
|
+
if (block.kind === "heading") {
|
|
1347
|
+
headingCount += 1;
|
|
1348
|
+
lines.push(`${"#".repeat(Math.max(2, Math.min(6, block.level)))} ${escapeInline(block.text)}`, "");
|
|
1349
|
+
continue;
|
|
1350
|
+
}
|
|
1351
|
+
textBlockCount += 1;
|
|
1352
|
+
if (block.kind === "list-item")
|
|
1353
|
+
lines.push(`- ${escapeInline(block.text)}`, "");
|
|
1354
|
+
else
|
|
1355
|
+
lines.push(escapeInline(block.text), "");
|
|
1356
|
+
}
|
|
1357
|
+
return {
|
|
1358
|
+
content: lines.join(`
|
|
1359
|
+
`).trim(),
|
|
1360
|
+
headingCount,
|
|
1361
|
+
textBlockCount
|
|
1362
|
+
};
|
|
1363
|
+
}
|
|
1364
|
+
function buildPdfMarkdown(options) {
|
|
1365
|
+
const filenameTitle = options.originalFilename.replace(/\.pdf$/iu, "").trim();
|
|
1366
|
+
const title = options.metadata.title ?? (filenameTitle === "" ? options.slug : filenameTitle);
|
|
1367
|
+
const body = markdownContent(options.blocks, options.images);
|
|
1368
|
+
const frontmatter = [
|
|
1369
|
+
"---",
|
|
1370
|
+
`title: ${yamlString(title)}`,
|
|
1371
|
+
`source: ${yamlString("source.pdf")}`,
|
|
1372
|
+
`source_type: ${yamlString("pdf")}`,
|
|
1373
|
+
`source_original_filename: ${yamlString(options.originalFilename)}`,
|
|
1374
|
+
`source_sha256: ${yamlString(options.sourceSha256)}`,
|
|
1375
|
+
...options.sourceUrl === undefined ? [] : [`source_url: ${yamlString(options.sourceUrl)}`],
|
|
1376
|
+
`pages: ${options.metadata.pageCount}`,
|
|
1377
|
+
`clipped: ${yamlString(options.capturedDate)}`,
|
|
1378
|
+
`capture_status: ${yamlString(options.status)}`,
|
|
1379
|
+
`capture_method: ${yamlString("poppler")}`,
|
|
1380
|
+
...options.metadata.author === null ? [] : [`author: ${yamlString(options.metadata.author)}`],
|
|
1381
|
+
...options.metadata.subject === null ? [] : [`description: ${yamlString(options.metadata.subject)}`],
|
|
1382
|
+
...options.metadata.createdAt === null ? [] : [`created: ${yamlString(options.metadata.createdAt)}`],
|
|
1383
|
+
...options.embeddedPlatforms.length === 0 ? [] : [`embedded_platforms: [${options.embeddedPlatforms.map(yamlString).join(", ")}]`],
|
|
1384
|
+
"---",
|
|
1385
|
+
"",
|
|
1386
|
+
`# ${escapeInline(title)}`,
|
|
1387
|
+
"",
|
|
1388
|
+
"[Open the source PDF](source.pdf)",
|
|
1389
|
+
"",
|
|
1390
|
+
body.content,
|
|
1391
|
+
""
|
|
1392
|
+
].join(`
|
|
1393
|
+
`);
|
|
1394
|
+
return {
|
|
1395
|
+
markdown: `${redactSensitiveText(frontmatter).trimEnd()}
|
|
1396
|
+
`,
|
|
1397
|
+
headingCount: body.headingCount,
|
|
1398
|
+
textBlockCount: body.textBlockCount
|
|
1399
|
+
};
|
|
1400
|
+
}
|
|
1401
|
+
|
|
1402
|
+
// src/pdf/ocr.ts
|
|
1403
|
+
function parseInteger(value) {
|
|
1404
|
+
if (value === undefined || !/^-?\d+$/u.test(value))
|
|
1405
|
+
return null;
|
|
1406
|
+
const parsed = Number(value);
|
|
1407
|
+
return Number.isSafeInteger(parsed) ? parsed : null;
|
|
1408
|
+
}
|
|
1409
|
+
function parseConfidence(value) {
|
|
1410
|
+
if (value === undefined)
|
|
1411
|
+
return null;
|
|
1412
|
+
const parsed = Number(value);
|
|
1413
|
+
return Number.isFinite(parsed) && parsed >= 0 && parsed <= 100 ? parsed : null;
|
|
1414
|
+
}
|
|
1415
|
+
function escapeMarkdownText(value) {
|
|
1416
|
+
return value.replace(/\\/gu, "\\\\").replace(/([`*_[\]{}<>#])/gu, "\\$1");
|
|
1417
|
+
}
|
|
1418
|
+
function parseTesseractTsv(value) {
|
|
1419
|
+
const lines = value.split(/\r?\n/gu);
|
|
1420
|
+
const header = lines[0]?.split("\t") ?? [];
|
|
1421
|
+
const positions = new Map(header.map((name, index) => [name, index]));
|
|
1422
|
+
const required = ["page_num", "block_num", "par_num", "line_num", "word_num", "conf", "text"];
|
|
1423
|
+
if (!required.every((name) => positions.has(name))) {
|
|
1424
|
+
throw new Error("Tesseract TSV is missing required columns");
|
|
1425
|
+
}
|
|
1426
|
+
const words = [];
|
|
1427
|
+
for (const row of lines.slice(1)) {
|
|
1428
|
+
if (row === "")
|
|
1429
|
+
continue;
|
|
1430
|
+
const fields = row.split("\t");
|
|
1431
|
+
const at = (name) => {
|
|
1432
|
+
const index = positions.get(name);
|
|
1433
|
+
return index === undefined ? undefined : fields[index];
|
|
1434
|
+
};
|
|
1435
|
+
const text2 = sanitizeTerminalText(at("text") ?? "").trim();
|
|
1436
|
+
const confidence2 = parseConfidence(at("conf"));
|
|
1437
|
+
const word = parseInteger(at("word_num"));
|
|
1438
|
+
const page = parseInteger(at("page_num"));
|
|
1439
|
+
const block = parseInteger(at("block_num"));
|
|
1440
|
+
const paragraph = parseInteger(at("par_num"));
|
|
1441
|
+
const line = parseInteger(at("line_num"));
|
|
1442
|
+
if (text2 === "" || confidence2 === null || word === null || page === null || block === null || paragraph === null || line === null)
|
|
1443
|
+
continue;
|
|
1444
|
+
words.push({
|
|
1445
|
+
page,
|
|
1446
|
+
block,
|
|
1447
|
+
paragraph,
|
|
1448
|
+
line,
|
|
1449
|
+
word,
|
|
1450
|
+
text: text2,
|
|
1451
|
+
confidence: confidence2
|
|
1452
|
+
});
|
|
1453
|
+
if (words.length >= 1e5)
|
|
1454
|
+
break;
|
|
1455
|
+
}
|
|
1456
|
+
words.sort((left, right) => left.page - right.page || left.block - right.block || left.paragraph - right.paragraph || left.line - right.line || left.word - right.word);
|
|
1457
|
+
const grouped = new Map;
|
|
1458
|
+
for (const word of words) {
|
|
1459
|
+
const lineIdentity = `${word.page}:${word.block}:${word.paragraph}:${word.line}`;
|
|
1460
|
+
const current = grouped.get(lineIdentity) ?? [];
|
|
1461
|
+
current.push(word.text);
|
|
1462
|
+
grouped.set(lineIdentity, current);
|
|
1463
|
+
}
|
|
1464
|
+
const textLines = [...grouped.values()].map((entries) => entries.join(" ").replace(/\s+/gu, " ").trim()).filter(Boolean);
|
|
1465
|
+
const text = redactSensitiveText(textLines.join(`
|
|
1466
|
+
`));
|
|
1467
|
+
const weightedCharacters = words.reduce((sum, word) => sum + word.text.length, 0);
|
|
1468
|
+
const confidence = weightedCharacters === 0 ? null : words.reduce((sum, word) => sum + word.confidence * word.text.length, 0) / weightedCharacters;
|
|
1469
|
+
const alphanumericCount = [...text].filter((character) => /[\p{Letter}\p{Number}]/u.test(character)).length;
|
|
1470
|
+
const substantive = words.length >= 3 && alphanumericCount >= 12 && confidence !== null && confidence >= 35;
|
|
1471
|
+
const markdown = substantive ? text.split(`
|
|
1472
|
+
`).map((line) => `> ${escapeMarkdownText(line)}`).join(`
|
|
1473
|
+
`) : "";
|
|
1474
|
+
return {
|
|
1475
|
+
text,
|
|
1476
|
+
markdown,
|
|
1477
|
+
confidence,
|
|
1478
|
+
wordCount: words.length,
|
|
1479
|
+
substantive
|
|
1480
|
+
};
|
|
1481
|
+
}
|
|
1482
|
+
async function ocrPdfImage(image, options) {
|
|
1483
|
+
if (options.tesseractPath === null) {
|
|
1484
|
+
return {
|
|
1485
|
+
kind: "visual",
|
|
1486
|
+
text: "",
|
|
1487
|
+
markdown: "",
|
|
1488
|
+
confidence: null,
|
|
1489
|
+
wordCount: 0,
|
|
1490
|
+
warnings: ["Tesseract is unavailable; retained the image without OCR text."]
|
|
1491
|
+
};
|
|
1492
|
+
}
|
|
1493
|
+
try {
|
|
1494
|
+
const result = await options.runTool({
|
|
1495
|
+
command: [
|
|
1496
|
+
options.tesseractPath,
|
|
1497
|
+
image.sourcePath,
|
|
1498
|
+
"stdout",
|
|
1499
|
+
"--psm",
|
|
1500
|
+
"6",
|
|
1501
|
+
"tsv"
|
|
1502
|
+
],
|
|
1503
|
+
timeoutMs: options.timeoutMs,
|
|
1504
|
+
maxOutputBytes: 16 * 1024 * 1024
|
|
1505
|
+
});
|
|
1506
|
+
if (result.exitCode !== 0) {
|
|
1507
|
+
return {
|
|
1508
|
+
kind: "visual",
|
|
1509
|
+
text: "",
|
|
1510
|
+
markdown: "",
|
|
1511
|
+
confidence: null,
|
|
1512
|
+
wordCount: 0,
|
|
1513
|
+
warnings: ["Tesseract could not read this image; retained it as visual evidence."]
|
|
1514
|
+
};
|
|
1515
|
+
}
|
|
1516
|
+
const parsed = parseTesseractTsv(result.stdout);
|
|
1517
|
+
return {
|
|
1518
|
+
kind: parsed.substantive ? "mixed" : "visual",
|
|
1519
|
+
text: parsed.text,
|
|
1520
|
+
markdown: parsed.markdown,
|
|
1521
|
+
confidence: parsed.confidence,
|
|
1522
|
+
wordCount: parsed.wordCount,
|
|
1523
|
+
warnings: parsed.substantive ? [] : ["Tesseract did not find sufficiently confident text; retained the image as visual evidence."]
|
|
1524
|
+
};
|
|
1525
|
+
} catch {
|
|
1526
|
+
return {
|
|
1527
|
+
kind: "visual",
|
|
1528
|
+
text: "",
|
|
1529
|
+
markdown: "",
|
|
1530
|
+
confidence: null,
|
|
1531
|
+
wordCount: 0,
|
|
1532
|
+
warnings: ["Tesseract OCR failed; retained the image as visual evidence."]
|
|
1533
|
+
};
|
|
1534
|
+
}
|
|
1535
|
+
}
|
|
1536
|
+
|
|
1537
|
+
// src/pdf/capture.ts
|
|
1538
|
+
import { createHash as createHash3 } from "crypto";
|
|
1539
|
+
import { mkdtempSync as mkdtempSync2, rmSync as rmSync2 } from "fs";
|
|
1540
|
+
import { tmpdir } from "os";
|
|
1541
|
+
import { join as join4 } from "path";
|
|
1542
|
+
function parsedInterpretations(raw, images) {
|
|
1543
|
+
const parsed = raw === undefined ? [] : parsePdfImageInterpretations(raw);
|
|
1544
|
+
const candidates = new Map(images.map((image) => [image.id, image]));
|
|
1545
|
+
const output = new Map;
|
|
1546
|
+
for (const interpretation of parsed) {
|
|
1547
|
+
const image = candidates.get(interpretation.id);
|
|
1548
|
+
if (image === undefined) {
|
|
1549
|
+
throw new Error(`PDF image annotation does not match an extracted image: ${interpretation.id}`);
|
|
1550
|
+
}
|
|
1551
|
+
if (image.sha256 !== interpretation.sha256) {
|
|
1552
|
+
throw new Error(`PDF image annotation hash does not match the extracted image: ${interpretation.id}`);
|
|
1553
|
+
}
|
|
1554
|
+
output.set(interpretation.id, interpretation);
|
|
1555
|
+
}
|
|
1556
|
+
return { byId: output, values: parsed };
|
|
1557
|
+
}
|
|
1558
|
+
function sortedImages(images) {
|
|
1559
|
+
return [...images].sort((left, right) => left.page - right.page || left.top - right.top || left.left - right.left || left.id.localeCompare(right.id));
|
|
1560
|
+
}
|
|
1561
|
+
function normalizedMetadata(value) {
|
|
1562
|
+
return value ?? null;
|
|
1563
|
+
}
|
|
1564
|
+
function resolvedAnnotation(image, annotation) {
|
|
1565
|
+
const markdown = annotation.kind === "visual" ? "" : annotation.markdown;
|
|
1566
|
+
return {
|
|
1567
|
+
image,
|
|
1568
|
+
kind: annotation.kind,
|
|
1569
|
+
method: annotation.method ?? "manual",
|
|
1570
|
+
markdown,
|
|
1571
|
+
alt: annotation.kind === "visual" ? annotation.alt ?? null : null,
|
|
1572
|
+
confidence: null,
|
|
1573
|
+
wordCount: annotation.kind === "visual" ? 0 : countWords(markdown),
|
|
1574
|
+
metadata: normalizedMetadata(annotation.metadata)
|
|
1575
|
+
};
|
|
1576
|
+
}
|
|
1577
|
+
function embeddedPlatforms(images) {
|
|
1578
|
+
const byIdentity = new Map;
|
|
1579
|
+
for (const image of images) {
|
|
1580
|
+
const platform = image.metadata?.platform?.trim();
|
|
1581
|
+
if (platform === undefined || platform === "")
|
|
1582
|
+
continue;
|
|
1583
|
+
const key = platform.toLocaleLowerCase("en-US");
|
|
1584
|
+
if (!byIdentity.has(key))
|
|
1585
|
+
byIdentity.set(key, platform);
|
|
1586
|
+
}
|
|
1587
|
+
return [...byIdentity.entries()].sort(([left], [right]) => left.localeCompare(right)).map(([, value]) => value);
|
|
1588
|
+
}
|
|
1589
|
+
function manifestImage(image) {
|
|
1590
|
+
return {
|
|
1591
|
+
id: image.image.id,
|
|
1592
|
+
page: image.image.page,
|
|
1593
|
+
top: image.image.top,
|
|
1594
|
+
left: image.image.left,
|
|
1595
|
+
width: image.image.width,
|
|
1596
|
+
height: image.image.height,
|
|
1597
|
+
asset: {
|
|
1598
|
+
path: pdfImageAssetPath(image.image),
|
|
1599
|
+
mimeType: image.image.mimeType,
|
|
1600
|
+
bytes: image.image.bytes,
|
|
1601
|
+
sha256: image.image.sha256
|
|
1602
|
+
},
|
|
1603
|
+
kind: image.kind,
|
|
1604
|
+
method: image.method,
|
|
1605
|
+
confidence: image.confidence,
|
|
1606
|
+
wordCount: image.wordCount,
|
|
1607
|
+
metadata: image.metadata
|
|
1608
|
+
};
|
|
1609
|
+
}
|
|
1610
|
+
async function runPdfCapture(options, dependencies = {}) {
|
|
1611
|
+
const callerOwnedWorkspace = options.workspaceDirectory !== undefined;
|
|
1612
|
+
const workspaceDirectory = options.workspaceDirectory ?? mkdtempSync2(join4(tmpdir(), "hraness-kb-pdf-"));
|
|
1613
|
+
try {
|
|
1614
|
+
const inspection = await inspectPdf({
|
|
1615
|
+
inputPath: options.inputPath,
|
|
1616
|
+
workspaceDirectory,
|
|
1617
|
+
...options.timeoutMs === undefined ? {} : { timeoutMs: options.timeoutMs },
|
|
1618
|
+
...options.maxPdfBytes === undefined ? {} : { maxPdfBytes: options.maxPdfBytes },
|
|
1619
|
+
...options.maxPages === undefined ? {} : { maxPages: options.maxPages },
|
|
1620
|
+
...options.maxImages === undefined ? {} : { maxImages: options.maxImages },
|
|
1621
|
+
...options.maxAssetBytes === undefined ? {} : { maxAssetBytes: options.maxAssetBytes },
|
|
1622
|
+
...options.maxTotalAssetBytes === undefined ? {} : { maxTotalAssetBytes: options.maxTotalAssetBytes }
|
|
1623
|
+
}, dependencies);
|
|
1624
|
+
const tools = resolvePdfTools(dependencies);
|
|
1625
|
+
const runTool = dependencies.runTool ?? runPdfToolCommand;
|
|
1626
|
+
const images = sortedImages(inspection.pages.flatMap((page) => page.images));
|
|
1627
|
+
const annotations = parsedInterpretations(options.interpretations, images);
|
|
1628
|
+
const resolvedImages = [];
|
|
1629
|
+
const warnings = [...inspection.warnings];
|
|
1630
|
+
let unclassifiedImages = 0;
|
|
1631
|
+
let ocrWarningCount = 0;
|
|
1632
|
+
for (const image of images) {
|
|
1633
|
+
const annotation = annotations.byId.get(image.id);
|
|
1634
|
+
if (annotation !== undefined) {
|
|
1635
|
+
resolvedImages.push(resolvedAnnotation(image, annotation));
|
|
1636
|
+
continue;
|
|
1637
|
+
}
|
|
1638
|
+
const ocr2 = await ocrPdfImage(image, {
|
|
1639
|
+
tesseractPath: tools.tesseract,
|
|
1640
|
+
timeoutMs: options.timeoutMs ?? 120000,
|
|
1641
|
+
runTool
|
|
1642
|
+
});
|
|
1643
|
+
if (tools.tesseract === null)
|
|
1644
|
+
unclassifiedImages += 1;
|
|
1645
|
+
for (const warning of ocr2.warnings) {
|
|
1646
|
+
warnings.push(`Page ${image.page} image ${image.id}: ${warning}`);
|
|
1647
|
+
ocrWarningCount += 1;
|
|
1648
|
+
}
|
|
1649
|
+
resolvedImages.push({
|
|
1650
|
+
image,
|
|
1651
|
+
kind: ocr2.kind,
|
|
1652
|
+
method: tools.tesseract === null ? "unclassified" : "tesseract",
|
|
1653
|
+
markdown: ocr2.markdown,
|
|
1654
|
+
alt: null,
|
|
1655
|
+
confidence: ocr2.confidence,
|
|
1656
|
+
wordCount: ocr2.wordCount,
|
|
1657
|
+
metadata: null
|
|
1658
|
+
});
|
|
1659
|
+
}
|
|
1660
|
+
const platforms = embeddedPlatforms(resolvedImages);
|
|
1661
|
+
const status = warnings.length > 0 || unclassifiedImages > 0 || ocrWarningCount > 0 ? "partial" : "complete";
|
|
1662
|
+
const filenameTitle = inspection.originalFilename.replace(/\.pdf$/iu, "").trim();
|
|
1663
|
+
const requestedSlug = options.slug ?? inspection.metadata.title ?? filenameTitle;
|
|
1664
|
+
const slug = slugify(sanitizeTerminalText(requestedSlug));
|
|
1665
|
+
if (slug === "")
|
|
1666
|
+
throw new Error("could not derive a safe PDF capture slug; pass an explicit slug");
|
|
1667
|
+
const blocks = layoutBlocks(inspection.pages);
|
|
1668
|
+
const byId = new Map(resolvedImages.map((image) => [image.image.id, image]));
|
|
1669
|
+
const capturedAt = (dependencies.now ?? (() => new Date))().toISOString();
|
|
1670
|
+
const built = buildPdfMarkdown({
|
|
1671
|
+
slug,
|
|
1672
|
+
originalFilename: inspection.originalFilename,
|
|
1673
|
+
sourceSha256: inspection.sourceSha256,
|
|
1674
|
+
...options.remoteSource === undefined ? {} : { sourceUrl: options.remoteSource.finalUrl },
|
|
1675
|
+
capturedDate: capturedAt.slice(0, 10),
|
|
1676
|
+
status,
|
|
1677
|
+
metadata: inspection.metadata,
|
|
1678
|
+
blocks,
|
|
1679
|
+
images: byId,
|
|
1680
|
+
embeddedPlatforms: platforms
|
|
1681
|
+
});
|
|
1682
|
+
const textImageCount = resolvedImages.filter((image) => image.kind === "text").length;
|
|
1683
|
+
const mixedImageCount = resolvedImages.filter((image) => image.kind === "mixed").length;
|
|
1684
|
+
const visualImageCount = resolvedImages.filter((image) => image.kind === "visual").length;
|
|
1685
|
+
const annotationCount = resolvedImages.filter((image) => image.method === "agent" || image.method === "manual").length;
|
|
1686
|
+
const tesseractCount = resolvedImages.filter((image) => image.method === "tesseract").length;
|
|
1687
|
+
const ocr = annotationCount > 0 && tesseractCount > 0 ? "mixed" : annotationCount > 0 ? "annotations" : tesseractCount > 0 ? "tesseract" : "unavailable";
|
|
1688
|
+
const finalWarnings = [...new Set(warnings)];
|
|
1689
|
+
const annotationsJson = annotations.values.length === 0 ? null : `${JSON.stringify(annotations.values, null, 2)}
|
|
1690
|
+
`;
|
|
1691
|
+
const annotationsManifest = annotationsJson === null ? null : {
|
|
1692
|
+
path: PDF_CAPTURE_ANNOTATIONS_FILENAME,
|
|
1693
|
+
count: annotations.values.length,
|
|
1694
|
+
bytes: Buffer.byteLength(annotationsJson),
|
|
1695
|
+
sha256: createHash3("sha256").update(annotationsJson).digest("hex")
|
|
1696
|
+
};
|
|
1697
|
+
const manifest = {
|
|
1698
|
+
schemaVersion: PDF_CAPTURE_MANIFEST_SCHEMA_VERSION,
|
|
1699
|
+
kind: "pdf",
|
|
1700
|
+
capturedAt,
|
|
1701
|
+
status,
|
|
1702
|
+
source: {
|
|
1703
|
+
originalFilename: sanitizeTerminalText(inspection.originalFilename).slice(0, 4096),
|
|
1704
|
+
path: PDF_CAPTURE_SOURCE_FILENAME,
|
|
1705
|
+
mimeType: "application/pdf",
|
|
1706
|
+
bytes: inspection.sourceBytes,
|
|
1707
|
+
sha256: inspection.sourceSha256,
|
|
1708
|
+
...options.remoteSource === undefined ? {} : {
|
|
1709
|
+
requestedUrl: options.remoteSource.requestedUrl,
|
|
1710
|
+
finalUrl: options.remoteSource.finalUrl
|
|
1711
|
+
}
|
|
1712
|
+
},
|
|
1713
|
+
document: {
|
|
1714
|
+
...inspection.metadata,
|
|
1715
|
+
processedPages: inspection.processedPages
|
|
1716
|
+
},
|
|
1717
|
+
extraction: {
|
|
1718
|
+
layout: "pdftohtml-xml",
|
|
1719
|
+
popplerVersion: inspection.popplerVersion,
|
|
1720
|
+
ocr,
|
|
1721
|
+
headingCount: built.headingCount,
|
|
1722
|
+
textBlockCount: built.textBlockCount,
|
|
1723
|
+
imageCount: resolvedImages.length,
|
|
1724
|
+
textImageCount,
|
|
1725
|
+
mixedImageCount,
|
|
1726
|
+
visualImageCount
|
|
1727
|
+
},
|
|
1728
|
+
images: resolvedImages.map(manifestImage),
|
|
1729
|
+
annotations: annotationsManifest,
|
|
1730
|
+
embeddedPlatforms: platforms,
|
|
1731
|
+
warnings: finalWarnings
|
|
1732
|
+
};
|
|
1733
|
+
const outputDirectory = persistPdfCapture({
|
|
1734
|
+
outputBase: options.outputBase,
|
|
1735
|
+
slug,
|
|
1736
|
+
force: options.force ?? false,
|
|
1737
|
+
sourcePath: inspection.inputPath,
|
|
1738
|
+
markdown: built.markdown,
|
|
1739
|
+
manifest,
|
|
1740
|
+
images,
|
|
1741
|
+
...annotationsJson === null ? {} : { annotationsJson }
|
|
1742
|
+
});
|
|
1743
|
+
return {
|
|
1744
|
+
status,
|
|
1745
|
+
slug,
|
|
1746
|
+
outputDirectory,
|
|
1747
|
+
markdownPath: join4(outputDirectory, pdfMarkdownFilename(slug)),
|
|
1748
|
+
sourcePath: join4(outputDirectory, PDF_CAPTURE_SOURCE_FILENAME),
|
|
1749
|
+
wordCount: countWords(built.markdown),
|
|
1750
|
+
pageCount: inspection.metadata.pageCount,
|
|
1751
|
+
processedPages: inspection.processedPages,
|
|
1752
|
+
imageCount: images.length,
|
|
1753
|
+
warnings: finalWarnings,
|
|
1754
|
+
markdown: built.markdown,
|
|
1755
|
+
manifest
|
|
1756
|
+
};
|
|
1757
|
+
} finally {
|
|
1758
|
+
if (!callerOwnedWorkspace)
|
|
1759
|
+
rmSync2(workspaceDirectory, { recursive: true, force: true });
|
|
1760
|
+
}
|
|
1761
|
+
}
|
|
1762
|
+
|
|
1763
|
+
// src/pdf/cli.ts
|
|
1764
|
+
import { lstatSync as lstatSync3, readFileSync as readFileSync3 } from "fs";
|
|
1765
|
+
import { resolve as resolve3 } from "path";
|
|
1766
|
+
|
|
1767
|
+
// src/pdf/source.ts
|
|
1768
|
+
import {
|
|
1769
|
+
chmodSync as chmodSync3,
|
|
1770
|
+
mkdtempSync as mkdtempSync3,
|
|
1771
|
+
rmSync as rmSync3,
|
|
1772
|
+
writeFileSync as writeFileSync2
|
|
1773
|
+
} from "fs";
|
|
1774
|
+
import { tmpdir as tmpdir2 } from "os";
|
|
1775
|
+
import { basename as basename2, join as join5 } from "path";
|
|
1776
|
+
var remoteUserAgent = "hraness-kb/0.7 PDF capture";
|
|
1777
|
+
function parseRemoteUrl(input) {
|
|
1778
|
+
if (!/^https?:\/\//iu.test(input))
|
|
1779
|
+
return null;
|
|
1780
|
+
let url;
|
|
1781
|
+
try {
|
|
1782
|
+
url = new URL(input);
|
|
1783
|
+
} catch {
|
|
1784
|
+
throw new Error("PDF URL is invalid");
|
|
1785
|
+
}
|
|
1786
|
+
if (url.protocol !== "http:" && url.protocol !== "https:") {
|
|
1787
|
+
throw new Error("PDF URL must use HTTP or HTTPS");
|
|
1788
|
+
}
|
|
1789
|
+
if (url.username !== "" || url.password !== "") {
|
|
1790
|
+
throw new Error("PDF URL must not contain embedded credentials");
|
|
1791
|
+
}
|
|
1792
|
+
return url;
|
|
1793
|
+
}
|
|
1794
|
+
function remoteFilename(url) {
|
|
1795
|
+
let decoded;
|
|
1796
|
+
try {
|
|
1797
|
+
decoded = decodeURIComponent(basename2(url.pathname));
|
|
1798
|
+
} catch {
|
|
1799
|
+
decoded = "source.pdf";
|
|
1800
|
+
}
|
|
1801
|
+
const normalized = decoded.normalize("NFKC").replace(/[^\p{Letter}\p{Number}._-]+/gu, "-").replace(/^[.-]+|[.-]+$/gu, "").slice(0, 180);
|
|
1802
|
+
const stem = normalized === "" ? "source" : normalized.replace(/\.pdf$/iu, "");
|
|
1803
|
+
return `${stem}.pdf`;
|
|
1804
|
+
}
|
|
1805
|
+
function assertPdfSignature(bytes) {
|
|
1806
|
+
if (bytes.byteLength < 5 || new TextDecoder().decode(bytes.subarray(0, 5)) !== "%PDF-") {
|
|
1807
|
+
throw new Error("remote PDF input does not have a valid PDF signature");
|
|
1808
|
+
}
|
|
1809
|
+
}
|
|
1810
|
+
async function preparePdfSource(input, options = {}, dependencies = {}) {
|
|
1811
|
+
const requestedUrl = parseRemoteUrl(input);
|
|
1812
|
+
if (requestedUrl === null) {
|
|
1813
|
+
return {
|
|
1814
|
+
inputPath: input,
|
|
1815
|
+
dispose: () => {}
|
|
1816
|
+
};
|
|
1817
|
+
}
|
|
1818
|
+
const result = await (dependencies.fetch ?? safeFetch)(requestedUrl, {
|
|
1819
|
+
timeoutMs: options.timeoutMs ?? pdfCaptureDefaults.timeoutMs,
|
|
1820
|
+
maxBytes: options.maxPdfBytes ?? pdfCaptureDefaults.maxPdfBytes,
|
|
1821
|
+
allowPrivateNetwork: false,
|
|
1822
|
+
userAgent: remoteUserAgent,
|
|
1823
|
+
accept: "application/pdf,application/octet-stream;q=0.9,*/*;q=0.1",
|
|
1824
|
+
retries: 2,
|
|
1825
|
+
maxRedirects: 5
|
|
1826
|
+
});
|
|
1827
|
+
assertPdfSignature(result.bytes);
|
|
1828
|
+
const makeTemporaryDirectory = dependencies.makeTemporaryDirectory ?? (() => mkdtempSync3(join5(tmpdir2(), "hraness-kb-pdf-source-")));
|
|
1829
|
+
const removeDirectory = dependencies.removeDirectory ?? ((path) => rmSync3(path, { recursive: true, force: true }));
|
|
1830
|
+
const directory = makeTemporaryDirectory();
|
|
1831
|
+
let disposed = false;
|
|
1832
|
+
const dispose = () => {
|
|
1833
|
+
if (disposed)
|
|
1834
|
+
return;
|
|
1835
|
+
disposed = true;
|
|
1836
|
+
removeDirectory(directory);
|
|
1837
|
+
};
|
|
1838
|
+
try {
|
|
1839
|
+
chmodSync3(directory, 448);
|
|
1840
|
+
const inputPath = join5(directory, remoteFilename(result.finalUrl));
|
|
1841
|
+
(dependencies.writeFile ?? writeFileSync2)(inputPath, result.bytes, {
|
|
1842
|
+
encoding: null,
|
|
1843
|
+
flag: "wx",
|
|
1844
|
+
mode: 384
|
|
1845
|
+
});
|
|
1846
|
+
return {
|
|
1847
|
+
inputPath,
|
|
1848
|
+
remoteSource: {
|
|
1849
|
+
requestedUrl: sanitizeArtifactUrl(requestedUrl.href),
|
|
1850
|
+
finalUrl: sanitizeArtifactUrl(result.finalUrl.href)
|
|
1851
|
+
},
|
|
1852
|
+
dispose
|
|
1853
|
+
};
|
|
1854
|
+
} catch (error) {
|
|
1855
|
+
dispose();
|
|
1856
|
+
throw error;
|
|
1857
|
+
}
|
|
1858
|
+
}
|
|
1859
|
+
|
|
1860
|
+
// src/pdf/cli.ts
|
|
1861
|
+
var defaultOutput = {
|
|
1862
|
+
stdout: (value) => process.stdout.write(value),
|
|
1863
|
+
stderr: (value) => process.stderr.write(value)
|
|
1864
|
+
};
|
|
1865
|
+
function safe(value) {
|
|
1866
|
+
return sanitizeTerminalLine(redactSensitiveText(value));
|
|
1867
|
+
}
|
|
1868
|
+
function terminalSafeJson(value) {
|
|
1869
|
+
return `${JSON.stringify(value, (_key, candidate) => typeof candidate === "string" ? sanitizeTerminalText(redactSensitiveText(candidate)) : candidate, 2)}
|
|
1870
|
+
`;
|
|
1871
|
+
}
|
|
1872
|
+
function readInterpretations(path) {
|
|
1873
|
+
const absolute = resolve3(path);
|
|
1874
|
+
const stats = lstatSync3(absolute);
|
|
1875
|
+
if (!stats.isFile() || stats.isSymbolicLink()) {
|
|
1876
|
+
throw new Error("PDF image annotations must be a regular JSON file");
|
|
1877
|
+
}
|
|
1878
|
+
if (stats.size > 16 * 1024 * 1024) {
|
|
1879
|
+
throw new Error("PDF image annotations exceed the 16MB limit");
|
|
1880
|
+
}
|
|
1881
|
+
let value;
|
|
1882
|
+
try {
|
|
1883
|
+
value = JSON.parse(readFileSync3(absolute, "utf8"));
|
|
1884
|
+
} catch {
|
|
1885
|
+
throw new Error("PDF image annotations are not valid JSON");
|
|
1886
|
+
}
|
|
1887
|
+
return parsePdfImageInterpretations(value);
|
|
1888
|
+
}
|
|
1889
|
+
function pdfCaptureSummary(outcome) {
|
|
1890
|
+
return {
|
|
1891
|
+
ok: true,
|
|
1892
|
+
status: outcome.status,
|
|
1893
|
+
slug: outcome.slug,
|
|
1894
|
+
outputDirectory: outcome.outputDirectory,
|
|
1895
|
+
markdownPath: outcome.markdownPath,
|
|
1896
|
+
sourcePath: outcome.sourcePath,
|
|
1897
|
+
wordCount: outcome.wordCount,
|
|
1898
|
+
pageCount: outcome.pageCount,
|
|
1899
|
+
processedPages: outcome.processedPages,
|
|
1900
|
+
imageCount: outcome.imageCount,
|
|
1901
|
+
warnings: outcome.warnings,
|
|
1902
|
+
manifest: outcome.manifest
|
|
1903
|
+
};
|
|
1904
|
+
}
|
|
1905
|
+
async function main(rawArguments = process.argv.slice(2), environment = process.env, output = defaultOutput, dependencies = {}) {
|
|
1906
|
+
const parsed = parsePdfArguments(rawArguments, environment);
|
|
1907
|
+
if (!parsed.ok) {
|
|
1908
|
+
output.stderr(`error: ${safe(parsed.message)}
|
|
1909
|
+
|
|
1910
|
+
${sanitizeTerminalText(pdfUsage)}`);
|
|
1911
|
+
return 2;
|
|
1912
|
+
}
|
|
1913
|
+
const arguments_ = parsed.value;
|
|
1914
|
+
if (arguments_.command === "help") {
|
|
1915
|
+
output.stdout(sanitizeTerminalText(pdfUsage));
|
|
1916
|
+
return 0;
|
|
1917
|
+
}
|
|
1918
|
+
if (!arguments_.quiet && !arguments_.json) {
|
|
1919
|
+
output.stderr(`Saving PDF ${safe(arguments_.input)} ...
|
|
1920
|
+
`);
|
|
1921
|
+
}
|
|
1922
|
+
let preparedSource = null;
|
|
1923
|
+
try {
|
|
1924
|
+
const interpretations = arguments_.interpretationsPath === undefined ? undefined : (dependencies.readInterpretations ?? readInterpretations)(arguments_.interpretationsPath);
|
|
1925
|
+
preparedSource = await (dependencies.preparePdfSource ?? preparePdfSource)(arguments_.input, {
|
|
1926
|
+
...arguments_.timeoutMs === undefined ? {} : { timeoutMs: arguments_.timeoutMs },
|
|
1927
|
+
...arguments_.maxPdfBytes === undefined ? {} : { maxPdfBytes: arguments_.maxPdfBytes }
|
|
1928
|
+
});
|
|
1929
|
+
const options = {
|
|
1930
|
+
inputPath: preparedSource.inputPath,
|
|
1931
|
+
outputBase: arguments_.outputBase,
|
|
1932
|
+
...preparedSource.remoteSource === undefined ? {} : { remoteSource: preparedSource.remoteSource },
|
|
1933
|
+
...arguments_.slug === undefined ? {} : { slug: arguments_.slug },
|
|
1934
|
+
...interpretations === undefined ? {} : { interpretations },
|
|
1935
|
+
force: arguments_.force,
|
|
1936
|
+
...arguments_.timeoutMs === undefined ? {} : { timeoutMs: arguments_.timeoutMs },
|
|
1937
|
+
...arguments_.maxPdfBytes === undefined ? {} : { maxPdfBytes: arguments_.maxPdfBytes },
|
|
1938
|
+
...arguments_.maxPages === undefined ? {} : { maxPages: arguments_.maxPages },
|
|
1939
|
+
...arguments_.maxImages === undefined ? {} : { maxImages: arguments_.maxImages },
|
|
1940
|
+
...arguments_.maxAssetBytes === undefined ? {} : { maxAssetBytes: arguments_.maxAssetBytes },
|
|
1941
|
+
...arguments_.maxTotalAssetBytes === undefined ? {} : { maxTotalAssetBytes: arguments_.maxTotalAssetBytes }
|
|
1942
|
+
};
|
|
1943
|
+
const outcome = await (dependencies.runPdfCapture ?? runPdfCapture)(options, dependencies.captureDependencies);
|
|
1944
|
+
if (arguments_.json)
|
|
1945
|
+
output.stdout(terminalSafeJson(pdfCaptureSummary(outcome)));
|
|
1946
|
+
else {
|
|
1947
|
+
output.stdout(`Done: ${safe(outcome.markdownPath)}
|
|
1948
|
+
`);
|
|
1949
|
+
const pages = outcome.processedPages === outcome.pageCount ? `${outcome.pageCount} pages` : `${outcome.processedPages} of ${outcome.pageCount} pages processed`;
|
|
1950
|
+
output.stdout(`Status: ${outcome.status}; ${pages}; ${outcome.wordCount} words; ${outcome.imageCount} images.
|
|
1951
|
+
`);
|
|
1952
|
+
}
|
|
1953
|
+
if (!arguments_.quiet && !arguments_.json) {
|
|
1954
|
+
for (const warning of outcome.warnings)
|
|
1955
|
+
output.stderr(`warning: ${safe(warning)}
|
|
1956
|
+
`);
|
|
1957
|
+
}
|
|
1958
|
+
return 0;
|
|
1959
|
+
} catch (error) {
|
|
1960
|
+
const message = safe(error instanceof Error ? error.message : String(error));
|
|
1961
|
+
if (arguments_.json)
|
|
1962
|
+
output.stdout(terminalSafeJson({ ok: false, error: message }));
|
|
1963
|
+
else
|
|
1964
|
+
output.stderr(`error: ${message}
|
|
1965
|
+
`);
|
|
1966
|
+
return 1;
|
|
1967
|
+
} finally {
|
|
1968
|
+
preparedSource?.dispose();
|
|
1969
|
+
}
|
|
1970
|
+
}
|
|
1971
|
+
if (false)
|
|
1972
|
+
;
|
|
1973
|
+
|
|
1974
|
+
export { pdfUsage, parsePdfArguments, decodePopplerText, parsePopplerXml, parsePdfInfo, layoutBlocks, parsePdfImageInterpretations, runPdfToolCommand, resolvePdfTools, pdfCaptureDefaults, inspectPdf, PDF_CAPTURE_MANIFEST_SCHEMA_VERSION, PDF_CAPTURE_MANIFEST_FILENAME, PDF_CAPTURE_SOURCE_FILENAME, PDF_CAPTURE_ANNOTATIONS_FILENAME, pdfImageAssetPath, persistPdfCapture, pdfMarkdownFilename, buildPdfMarkdown, parseTesseractTsv, ocrPdfImage, runPdfCapture, pdfCaptureSummary, main };
|