@hraness/kb 0.17.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (200) hide show
  1. package/DISCLOSURE +43 -0
  2. package/LICENSE +21 -0
  3. package/README.md +545 -0
  4. package/dist/agent-context.js +35 -0
  5. package/dist/agent-guide-audit.js +18 -0
  6. package/dist/attachments.js +19 -0
  7. package/dist/authoring.js +33 -0
  8. package/dist/benchmark.js +19 -0
  9. package/dist/capture.js +49 -0
  10. package/dist/cli.js +3646 -0
  11. package/dist/clip/acquire.js +57 -0
  12. package/dist/clip/args.js +21 -0
  13. package/dist/clip/bounded-byte-buffer.js +9 -0
  14. package/dist/clip/browser-profiles.js +17 -0
  15. package/dist/clip/bundle-reader.js +11 -0
  16. package/dist/clip/cli.js +27 -0
  17. package/dist/clip/cookies.js +21 -0
  18. package/dist/clip/defuddle-worker.js +84 -0
  19. package/dist/clip/doctor.js +32 -0
  20. package/dist/clip/jobs.js +784 -0
  21. package/dist/clip/network-proxy.js +9 -0
  22. package/dist/clip/network.js +28 -0
  23. package/dist/clip/persist.js +30 -0
  24. package/dist/clip/refresh.js +11 -0
  25. package/dist/clip/terminal.js +11 -0
  26. package/dist/evaluation-builder.js +12238 -0
  27. package/dist/evaluation-kb.js +26 -0
  28. package/dist/evaluation.js +35 -0
  29. package/dist/git.js +39 -0
  30. package/dist/graph.js +39 -0
  31. package/dist/index-01jj6rbv.js +1103 -0
  32. package/dist/index-06c9ctr6.js +738 -0
  33. package/dist/index-0kavxzqj.js +2480 -0
  34. package/dist/index-1gwbassd.js +743 -0
  35. package/dist/index-1n418kb9.js +827 -0
  36. package/dist/index-1vrd1rmn.js +56 -0
  37. package/dist/index-1xxnjn0d.js +171 -0
  38. package/dist/index-3rm7cz6h.js +373 -0
  39. package/dist/index-3v2z4f0q.js +328 -0
  40. package/dist/index-48pz4jpc.js +392 -0
  41. package/dist/index-4j3tt0c3.js +234 -0
  42. package/dist/index-5n05se68.js +307 -0
  43. package/dist/index-5vwpzb5a.js +764 -0
  44. package/dist/index-6g2pv9d2.js +370 -0
  45. package/dist/index-7fzc32gf.js +712 -0
  46. package/dist/index-84x0vjjp.js +432 -0
  47. package/dist/index-adx6khj5.js +488 -0
  48. package/dist/index-b88v3vtm.js +805 -0
  49. package/dist/index-cv6fh7z5.js +462 -0
  50. package/dist/index-cxfrakt7.js +1067 -0
  51. package/dist/index-d13v9ckt.js +150 -0
  52. package/dist/index-dycjbhw2.js +1943 -0
  53. package/dist/index-dyqwejk5.js +531 -0
  54. package/dist/index-e5fbsywq.js +658 -0
  55. package/dist/index-f984hw45.js +927 -0
  56. package/dist/index-gh719d91.js +63 -0
  57. package/dist/index-hgve9rh2.js +2046 -0
  58. package/dist/index-hya40gb2.js +474 -0
  59. package/dist/index-j4zgmzjr.js +152 -0
  60. package/dist/index-jsmvyyvf.js +1166 -0
  61. package/dist/index-kh0pwvdd.js +1974 -0
  62. package/dist/index-mqx4nd6v.js +119 -0
  63. package/dist/index-mxxxytys.js +1254 -0
  64. package/dist/index-n5dd7r0v.js +711 -0
  65. package/dist/index-npg9z1a4.js +431 -0
  66. package/dist/index-pj501bh1.js +140 -0
  67. package/dist/index-qry4vhxk.js +1041 -0
  68. package/dist/index-s2gw5aw9.js +199 -0
  69. package/dist/index-vvdj6kca.js +70 -0
  70. package/dist/index-w2zc0vwa.js +571 -0
  71. package/dist/index-x3fthpsc.js +585 -0
  72. package/dist/index-zr53sf63.js +44 -0
  73. package/dist/index-zxdy5pby.js +2728 -0
  74. package/dist/index-zzhgcwyt.js +966 -0
  75. package/dist/index.js +453 -0
  76. package/dist/navigation.js +13 -0
  77. package/dist/pdf.js +85 -0
  78. package/dist/percolate.js +24 -0
  79. package/dist/portfolio.js +77 -0
  80. package/dist/query.js +29 -0
  81. package/dist/repository-memory.js +60 -0
  82. package/dist/sdk.js +36 -0
  83. package/dist/search-rules.js +34 -0
  84. package/dist/search.js +22 -0
  85. package/dist/semantic.js +39 -0
  86. package/dist/source-inbox.js +17 -0
  87. package/dist/untrusted-content.js +32 -0
  88. package/dist/url-intelligence.js +74 -0
  89. package/dist/workflow.js +23 -0
  90. package/dist/workflows/decision-context.js +20 -0
  91. package/dist/workflows/explain-change.js +8 -0
  92. package/dist/workflows/index.js +28 -0
  93. package/dist/workflows/plan-radar.js +10 -0
  94. package/package.json +409 -0
  95. package/skills/kb/AGENTS.md +19 -0
  96. package/skills/kb/SKILL.md +82 -0
  97. package/skills/kb/agents/openai.yaml +4 -0
  98. package/skills/kb/references/pdf-review.md +51 -0
  99. package/skills/kb/references/percolate.md +144 -0
  100. package/skills/kb/references/plan-structure.md +105 -0
  101. package/skills/kb/references/plan.md +115 -0
  102. package/skills/kb/references/query.md +231 -0
  103. package/skills/kb/references/refresh.md +133 -0
  104. package/skills/kb/references/save-pdf.md +125 -0
  105. package/skills/kb/references/save-url.md +178 -0
  106. package/skills/kb/references/url-authentication.md +75 -0
  107. package/skills/kb/references/url-platforms.md +25 -0
  108. package/src/agent-context.ts +1241 -0
  109. package/src/agent-guide-audit.ts +750 -0
  110. package/src/attachments.ts +723 -0
  111. package/src/authoring.ts +1725 -0
  112. package/src/benchmark.ts +362 -0
  113. package/src/capture.ts +68 -0
  114. package/src/cli.ts +3660 -0
  115. package/src/clip/acquire.ts +1273 -0
  116. package/src/clip/archive-today.ts +402 -0
  117. package/src/clip/args.ts +423 -0
  118. package/src/clip/assets.ts +262 -0
  119. package/src/clip/bounded-byte-buffer.ts +75 -0
  120. package/src/clip/browser-profiles.ts +372 -0
  121. package/src/clip/bundle-reader.ts +608 -0
  122. package/src/clip/capture.ts +1054 -0
  123. package/src/clip/cli.ts +181 -0
  124. package/src/clip/cookies.ts +512 -0
  125. package/src/clip/doctor.ts +1110 -0
  126. package/src/clip/extract.ts +868 -0
  127. package/src/clip/jobs.ts +959 -0
  128. package/src/clip/lib.ts +861 -0
  129. package/src/clip/media.ts +1474 -0
  130. package/src/clip/metadata-search-tool/Cargo.lock +2498 -0
  131. package/src/clip/metadata-search-tool/Cargo.toml +18 -0
  132. package/src/clip/metadata-search-tool/runner.ts +685 -0
  133. package/src/clip/metadata-search-tool/src/main.rs +389 -0
  134. package/src/clip/metadata-search.ts +687 -0
  135. package/src/clip/network-proxy.ts +627 -0
  136. package/src/clip/network.ts +950 -0
  137. package/src/clip/package-root.ts +54 -0
  138. package/src/clip/persist.ts +1472 -0
  139. package/src/clip/platforms.ts +1824 -0
  140. package/src/clip/refresh.ts +197 -0
  141. package/src/clip/structured.ts +432 -0
  142. package/src/clip/terminal.ts +197 -0
  143. package/src/clip/url-intelligence.ts +581 -0
  144. package/src/clip/url-metadata-backfill.ts +616 -0
  145. package/src/clip/url-metadata-cli.ts +251 -0
  146. package/src/clip/url-metadata.ts +1397 -0
  147. package/src/evaluation-analysis-v2.ts +1440 -0
  148. package/src/evaluation-builder.ts +2539 -0
  149. package/src/evaluation-corpus-authoring.ts +2052 -0
  150. package/src/evaluation-evidence.ts +1147 -0
  151. package/src/evaluation-execution-v2.ts +426 -0
  152. package/src/evaluation-implementation.ts +131 -0
  153. package/src/evaluation-kb-closure.ts +1430 -0
  154. package/src/evaluation-kb-runner-v2.ts +735 -0
  155. package/src/evaluation-kb-v2.ts +2567 -0
  156. package/src/evaluation-kb.ts +1018 -0
  157. package/src/evaluation-measurement-v2.ts +1462 -0
  158. package/src/evaluation-packing-v2.ts +452 -0
  159. package/src/evaluation-redaction.ts +28 -0
  160. package/src/evaluation-v2.ts +3857 -0
  161. package/src/evaluation.ts +1220 -0
  162. package/src/git.ts +1276 -0
  163. package/src/graph.ts +1487 -0
  164. package/src/index.ts +20 -0
  165. package/src/init.ts +123 -0
  166. package/src/navigation.ts +277 -0
  167. package/src/note-lock.ts +529 -0
  168. package/src/pdf/args.ts +202 -0
  169. package/src/pdf/capture.ts +298 -0
  170. package/src/pdf/cli.ts +159 -0
  171. package/src/pdf/extract.ts +303 -0
  172. package/src/pdf/layout.ts +618 -0
  173. package/src/pdf/markdown.ts +154 -0
  174. package/src/pdf/model.ts +234 -0
  175. package/src/pdf/ocr.ts +190 -0
  176. package/src/pdf/persist.ts +314 -0
  177. package/src/pdf/source.ts +135 -0
  178. package/src/pdf/tools.ts +146 -0
  179. package/src/pdf.ts +92 -0
  180. package/src/percolate.ts +877 -0
  181. package/src/portfolio-audit.ts +574 -0
  182. package/src/portfolio-identity.ts +162 -0
  183. package/src/portfolio-registry.ts +473 -0
  184. package/src/portfolio.ts +765 -0
  185. package/src/query.ts +646 -0
  186. package/src/repository-memory.ts +1150 -0
  187. package/src/sdk.ts +1452 -0
  188. package/src/search-rules.ts +712 -0
  189. package/src/search.ts +710 -0
  190. package/src/semantic-runtime.ts +1139 -0
  191. package/src/semantic.ts +2480 -0
  192. package/src/source-inbox.ts +232 -0
  193. package/src/untrusted-content.ts +401 -0
  194. package/src/url-intelligence.ts +104 -0
  195. package/src/vault.ts +636 -0
  196. package/src/workflow.ts +653 -0
  197. package/src/workflows/decision-context.ts +81 -0
  198. package/src/workflows/explain-change.ts +73 -0
  199. package/src/workflows/index.ts +3 -0
  200. package/src/workflows/plan-radar.ts +101 -0
@@ -0,0 +1,202 @@
1
+ export const pdfUsage = `kb pdf — save a local or public remote PDF as an auditable Markdown bundle
2
+
3
+ Usage:
4
+ kb pdf <file-or-url> [--output <directory>] [--slug <slug>] [--annotations <json>] [--force] [--json]
5
+ kb pdf save <file-or-url> [capture options]
6
+
7
+ Capture options:
8
+ --output <directory> Bundle parent (default: KB_PDF_OUTPUT or kb/articles)
9
+ --slug <slug> Override the title-derived bundle name
10
+ --annotations <json> Optional hash-bound image interpretations from an agent
11
+ --timeout-ms <milliseconds> Per-tool timeout
12
+ --max-pdf-bytes <size> Input limit, for example 512mb
13
+ --max-pages <count> Maximum pages to process
14
+ --max-images <count> Maximum embedded images
15
+ --max-asset-bytes <size> Per-image limit
16
+ --max-total-asset-bytes <size>
17
+ --force Replace only a compatible PDF capture bundle
18
+ --json Emit a machine-readable result
19
+ --quiet Suppress progress and warning lines
20
+ `;
21
+
22
+ export type PdfCliArguments =
23
+ | { readonly command: "help" }
24
+ | {
25
+ readonly command: "capture";
26
+ readonly input: string;
27
+ readonly outputBase: string;
28
+ readonly slug?: string;
29
+ readonly interpretationsPath?: string;
30
+ readonly force: boolean;
31
+ readonly json: boolean;
32
+ readonly quiet: boolean;
33
+ readonly timeoutMs?: number;
34
+ readonly maxPdfBytes?: number;
35
+ readonly maxPages?: number;
36
+ readonly maxImages?: number;
37
+ readonly maxAssetBytes?: number;
38
+ readonly maxTotalAssetBytes?: number;
39
+ };
40
+
41
+ export type ParsePdfArgumentsResult =
42
+ | { readonly ok: true; readonly value: PdfCliArguments }
43
+ | { readonly ok: false; readonly message: string };
44
+
45
+ const valueOptions = new Set([
46
+ "--output",
47
+ "--slug",
48
+ "--annotations",
49
+ "--timeout-ms",
50
+ "--max-pdf-bytes",
51
+ "--max-pages",
52
+ "--max-images",
53
+ "--max-asset-bytes",
54
+ "--max-total-asset-bytes",
55
+ ]);
56
+
57
+ function optionValue(
58
+ arguments_: readonly string[],
59
+ index: number,
60
+ name: string,
61
+ ): string | ParsePdfArgumentsResult {
62
+ const value = arguments_[index + 1];
63
+ return value === undefined || value.startsWith("--")
64
+ ? { ok: false, message: `${name} requires a value` }
65
+ : value;
66
+ }
67
+
68
+ function positiveInteger(value: string, name: string, maximum: number): number | string {
69
+ if (!/^\d+$/u.test(value)) return `${name} must be a positive integer`;
70
+ const parsed = Number(value);
71
+ return Number.isSafeInteger(parsed) && parsed >= 1 && parsed <= maximum
72
+ ? parsed
73
+ : `${name} must be between 1 and ${maximum}`;
74
+ }
75
+
76
+ function byteSize(value: string, name: string, maximum: number): number | string {
77
+ const match = /^(\d+)(b|kb|mb|gb)?$/iu.exec(value);
78
+ if (match === null || match[1] === undefined) {
79
+ return `${name} must be an integer byte size such as 500000, 25mb, or 1gb`;
80
+ }
81
+ const amount = Number(match[1]);
82
+ const unit = match[2]?.toLowerCase() ?? "b";
83
+ const multiplier = unit === "gb"
84
+ ? 1024 ** 3
85
+ : unit === "mb"
86
+ ? 1024 ** 2
87
+ : unit === "kb"
88
+ ? 1024
89
+ : 1;
90
+ const parsed = amount * multiplier;
91
+ return Number.isSafeInteger(parsed) && parsed >= 1 && parsed <= maximum
92
+ ? parsed
93
+ : `${name} must be between 1 byte and ${maximum} bytes`;
94
+ }
95
+
96
+ /** Parse the delegated `kb pdf` surface without touching the filesystem. */
97
+ export function parsePdfArguments(
98
+ rawArguments: readonly string[],
99
+ environment: Readonly<Record<string, string | undefined>> = {},
100
+ ): ParsePdfArgumentsResult {
101
+ if (
102
+ rawArguments.length === 0
103
+ || rawArguments[0] === "help"
104
+ || rawArguments[0] === "--help"
105
+ || rawArguments[0] === "-h"
106
+ ) return { ok: true, value: { command: "help" } };
107
+
108
+ let cursor = rawArguments[0] === "save" || rawArguments[0] === "capture" ? 1 : 0;
109
+ const positional: string[] = [];
110
+ let outputBase = environment.KB_PDF_OUTPUT ?? "kb/articles";
111
+ let slug: string | undefined;
112
+ let interpretationsPath: string | undefined;
113
+ let force = false;
114
+ let json = false;
115
+ let quiet = false;
116
+ let timeoutMs: number | undefined;
117
+ let maxPdfBytes: number | undefined;
118
+ let maxPages: number | undefined;
119
+ let maxImages: number | undefined;
120
+ let maxAssetBytes: number | undefined;
121
+ let maxTotalAssetBytes: number | undefined;
122
+
123
+ for (; cursor < rawArguments.length; cursor += 1) {
124
+ const argument = rawArguments[cursor];
125
+ if (argument === undefined) continue;
126
+ if (!argument.startsWith("--")) {
127
+ positional.push(argument);
128
+ continue;
129
+ }
130
+ if (argument === "--force") {
131
+ force = true;
132
+ continue;
133
+ }
134
+ if (argument === "--json") {
135
+ json = true;
136
+ continue;
137
+ }
138
+ if (argument === "--quiet") {
139
+ quiet = true;
140
+ continue;
141
+ }
142
+ if (!valueOptions.has(argument)) {
143
+ return { ok: false, message: `unknown PDF option: ${argument}` };
144
+ }
145
+ const rawValue = optionValue(rawArguments, cursor, argument);
146
+ if (typeof rawValue !== "string") return rawValue;
147
+ cursor += 1;
148
+ if (argument === "--output") outputBase = rawValue;
149
+ else if (argument === "--slug") slug = rawValue;
150
+ else if (argument === "--annotations") interpretationsPath = rawValue;
151
+ else if (argument === "--timeout-ms") {
152
+ const parsed = positiveInteger(rawValue, argument, 10 * 60_000);
153
+ if (typeof parsed === "string") return { ok: false, message: parsed };
154
+ timeoutMs = parsed;
155
+ } else if (argument === "--max-pages") {
156
+ const parsed = positiveInteger(rawValue, argument, 10_000);
157
+ if (typeof parsed === "string") return { ok: false, message: parsed };
158
+ maxPages = parsed;
159
+ } else if (argument === "--max-images") {
160
+ const parsed = positiveInteger(rawValue, argument, 10_000);
161
+ if (typeof parsed === "string") return { ok: false, message: parsed };
162
+ maxImages = parsed;
163
+ } else {
164
+ const maximum = argument === "--max-asset-bytes" ? 2 * 1024 ** 3 : 8 * 1024 ** 3;
165
+ const parsed = byteSize(rawValue, argument, maximum);
166
+ if (typeof parsed === "string") return { ok: false, message: parsed };
167
+ if (argument === "--max-pdf-bytes") maxPdfBytes = parsed;
168
+ else if (argument === "--max-asset-bytes") maxAssetBytes = parsed;
169
+ else maxTotalAssetBytes = parsed;
170
+ }
171
+ }
172
+
173
+ const input = positional[0];
174
+ if (input === undefined || positional.length !== 1) {
175
+ return { ok: false, message: "kb pdf requires exactly one PDF path or public URL" };
176
+ }
177
+ if (input.length > 64 * 1024) {
178
+ return { ok: false, message: "PDF input exceeds the 65536 code-unit limit" };
179
+ }
180
+ if (outputBase.trim() === "") return { ok: false, message: "--output must not be empty" };
181
+ if (slug !== undefined && slug.trim() === "") return { ok: false, message: "--slug must not be empty" };
182
+
183
+ return {
184
+ ok: true,
185
+ value: {
186
+ command: "capture",
187
+ input,
188
+ outputBase,
189
+ ...(slug === undefined ? {} : { slug }),
190
+ ...(interpretationsPath === undefined ? {} : { interpretationsPath }),
191
+ force,
192
+ json,
193
+ quiet,
194
+ ...(timeoutMs === undefined ? {} : { timeoutMs }),
195
+ ...(maxPdfBytes === undefined ? {} : { maxPdfBytes }),
196
+ ...(maxPages === undefined ? {} : { maxPages }),
197
+ ...(maxImages === undefined ? {} : { maxImages }),
198
+ ...(maxAssetBytes === undefined ? {} : { maxAssetBytes }),
199
+ ...(maxTotalAssetBytes === undefined ? {} : { maxTotalAssetBytes }),
200
+ },
201
+ };
202
+ }
@@ -0,0 +1,298 @@
1
+ import { createHash } from "node:crypto";
2
+ import { mkdtempSync, rmSync } from "node:fs";
3
+ import { tmpdir } from "node:os";
4
+ import { join } from "node:path";
5
+
6
+ import { countWords } from "../clip/extract.js";
7
+ import { slugify } from "../clip/lib.js";
8
+ import { sanitizeTerminalText } from "../clip/terminal.js";
9
+ import { inspectPdf } from "./extract.js";
10
+ import {
11
+ layoutBlocks,
12
+ parsePdfImageInterpretations,
13
+ } from "./layout.js";
14
+ import {
15
+ buildPdfMarkdown,
16
+ type ResolvedPdfImage,
17
+ } from "./markdown.js";
18
+ import type {
19
+ PdfCaptureDependencies,
20
+ PdfCaptureManifest,
21
+ PdfCaptureOptions,
22
+ PdfCaptureOutcome,
23
+ PdfImageCandidate,
24
+ PdfImageInterpretation,
25
+ PdfImageSemanticMetadata,
26
+ PdfManifestImage,
27
+ } from "./model.js";
28
+ import {
29
+ PDF_CAPTURE_MANIFEST_SCHEMA_VERSION,
30
+ PDF_CAPTURE_ANNOTATIONS_FILENAME,
31
+ PDF_CAPTURE_SOURCE_FILENAME,
32
+ } from "./model.js";
33
+ import { ocrPdfImage } from "./ocr.js";
34
+ import {
35
+ pdfImageAssetPath,
36
+ pdfMarkdownFilename,
37
+ persistPdfCapture,
38
+ } from "./persist.js";
39
+ import { resolvePdfTools, runPdfToolCommand } from "./tools.js";
40
+
41
+ function parsedInterpretations(
42
+ raw: readonly PdfImageInterpretation[] | undefined,
43
+ images: readonly PdfImageCandidate[],
44
+ ): {
45
+ readonly byId: ReadonlyMap<string, PdfImageInterpretation>;
46
+ readonly values: readonly PdfImageInterpretation[];
47
+ } {
48
+ const parsed = raw === undefined ? [] : parsePdfImageInterpretations(raw);
49
+ const candidates = new Map(images.map((image) => [image.id, image]));
50
+ const output = new Map<string, PdfImageInterpretation>();
51
+ for (const interpretation of parsed) {
52
+ const image = candidates.get(interpretation.id);
53
+ if (image === undefined) {
54
+ throw new Error(`PDF image annotation does not match an extracted image: ${interpretation.id}`);
55
+ }
56
+ if (image.sha256 !== interpretation.sha256) {
57
+ throw new Error(`PDF image annotation hash does not match the extracted image: ${interpretation.id}`);
58
+ }
59
+ output.set(interpretation.id, interpretation);
60
+ }
61
+ return { byId: output, values: parsed };
62
+ }
63
+
64
+ function sortedImages(images: readonly PdfImageCandidate[]): readonly PdfImageCandidate[] {
65
+ return [...images].sort((left, right) =>
66
+ left.page - right.page
67
+ || left.top - right.top
68
+ || left.left - right.left
69
+ || left.id.localeCompare(right.id));
70
+ }
71
+
72
+ function normalizedMetadata(
73
+ value: PdfImageSemanticMetadata | undefined,
74
+ ): PdfImageSemanticMetadata | null {
75
+ return value ?? null;
76
+ }
77
+
78
+ function resolvedAnnotation(
79
+ image: PdfImageCandidate,
80
+ annotation: PdfImageInterpretation,
81
+ ): ResolvedPdfImage {
82
+ const markdown = annotation.kind === "visual" ? "" : annotation.markdown;
83
+ return {
84
+ image,
85
+ kind: annotation.kind,
86
+ method: annotation.method ?? "manual",
87
+ markdown,
88
+ alt: annotation.kind === "visual" ? (annotation.alt ?? null) : null,
89
+ confidence: null,
90
+ wordCount: annotation.kind === "visual" ? 0 : countWords(markdown),
91
+ metadata: normalizedMetadata(annotation.metadata),
92
+ };
93
+ }
94
+
95
+ function embeddedPlatforms(images: readonly ResolvedPdfImage[]): readonly string[] {
96
+ const byIdentity = new Map<string, string>();
97
+ for (const image of images) {
98
+ const platform = image.metadata?.platform?.trim();
99
+ if (platform === undefined || platform === "") continue;
100
+ const key = platform.toLocaleLowerCase("en-US");
101
+ if (!byIdentity.has(key)) byIdentity.set(key, platform);
102
+ }
103
+ return [...byIdentity.entries()]
104
+ .sort(([left], [right]) => left.localeCompare(right))
105
+ .map(([, value]) => value);
106
+ }
107
+
108
+ function manifestImage(image: ResolvedPdfImage): PdfManifestImage {
109
+ return {
110
+ id: image.image.id,
111
+ page: image.image.page,
112
+ top: image.image.top,
113
+ left: image.image.left,
114
+ width: image.image.width,
115
+ height: image.image.height,
116
+ asset: {
117
+ path: pdfImageAssetPath(image.image),
118
+ mimeType: image.image.mimeType,
119
+ bytes: image.image.bytes,
120
+ sha256: image.image.sha256,
121
+ },
122
+ kind: image.kind,
123
+ method: image.method,
124
+ confidence: image.confidence,
125
+ wordCount: image.wordCount,
126
+ metadata: image.metadata,
127
+ };
128
+ }
129
+
130
+ /** Inspect, OCR/annotate, render, and atomically persist one prepared PDF source. */
131
+ export async function runPdfCapture(
132
+ options: PdfCaptureOptions,
133
+ dependencies: PdfCaptureDependencies = {},
134
+ ): Promise<PdfCaptureOutcome> {
135
+ const callerOwnedWorkspace = options.workspaceDirectory !== undefined;
136
+ const workspaceDirectory = options.workspaceDirectory
137
+ ?? mkdtempSync(join(tmpdir(), "hraness-kb-pdf-"));
138
+ try {
139
+ const inspection = await inspectPdf({
140
+ inputPath: options.inputPath,
141
+ workspaceDirectory,
142
+ ...(options.timeoutMs === undefined ? {} : { timeoutMs: options.timeoutMs }),
143
+ ...(options.maxPdfBytes === undefined ? {} : { maxPdfBytes: options.maxPdfBytes }),
144
+ ...(options.maxPages === undefined ? {} : { maxPages: options.maxPages }),
145
+ ...(options.maxImages === undefined ? {} : { maxImages: options.maxImages }),
146
+ ...(options.maxAssetBytes === undefined ? {} : { maxAssetBytes: options.maxAssetBytes }),
147
+ ...(options.maxTotalAssetBytes === undefined ? {} : { maxTotalAssetBytes: options.maxTotalAssetBytes }),
148
+ }, dependencies);
149
+ const tools = resolvePdfTools(dependencies);
150
+ const runTool = dependencies.runTool ?? runPdfToolCommand;
151
+ const images = sortedImages(inspection.pages.flatMap((page) => page.images));
152
+ const annotations = parsedInterpretations(options.interpretations, images);
153
+ const resolvedImages: ResolvedPdfImage[] = [];
154
+ const warnings = [...inspection.warnings];
155
+ let unclassifiedImages = 0;
156
+ let ocrWarningCount = 0;
157
+
158
+ for (const image of images) {
159
+ const annotation = annotations.byId.get(image.id);
160
+ if (annotation !== undefined) {
161
+ resolvedImages.push(resolvedAnnotation(image, annotation));
162
+ continue;
163
+ }
164
+ const ocr = await ocrPdfImage(image, {
165
+ tesseractPath: tools.tesseract,
166
+ timeoutMs: options.timeoutMs ?? 120_000,
167
+ runTool,
168
+ });
169
+ if (tools.tesseract === null) unclassifiedImages += 1;
170
+ for (const warning of ocr.warnings) {
171
+ warnings.push(`Page ${image.page} image ${image.id}: ${warning}`);
172
+ ocrWarningCount += 1;
173
+ }
174
+ resolvedImages.push({
175
+ image,
176
+ kind: ocr.kind,
177
+ method: tools.tesseract === null ? "unclassified" : "tesseract",
178
+ markdown: ocr.markdown,
179
+ alt: null,
180
+ confidence: ocr.confidence,
181
+ wordCount: ocr.wordCount,
182
+ metadata: null,
183
+ });
184
+ }
185
+ const platforms = embeddedPlatforms(resolvedImages);
186
+ const status = warnings.length > 0 || unclassifiedImages > 0 || ocrWarningCount > 0
187
+ ? "partial"
188
+ : "complete";
189
+ const filenameTitle = inspection.originalFilename.replace(/\.pdf$/iu, "").trim();
190
+ const requestedSlug = options.slug ?? inspection.metadata.title ?? filenameTitle;
191
+ const slug = slugify(sanitizeTerminalText(requestedSlug));
192
+ if (slug === "") throw new Error("could not derive a safe PDF capture slug; pass an explicit slug");
193
+ const blocks = layoutBlocks(inspection.pages);
194
+ const byId = new Map(resolvedImages.map((image) => [image.image.id, image]));
195
+ const capturedAt = (dependencies.now ?? (() => new Date()))().toISOString();
196
+ const built = buildPdfMarkdown({
197
+ slug,
198
+ originalFilename: inspection.originalFilename,
199
+ sourceSha256: inspection.sourceSha256,
200
+ ...(options.remoteSource === undefined ? {} : { sourceUrl: options.remoteSource.finalUrl }),
201
+ capturedDate: capturedAt.slice(0, 10),
202
+ status,
203
+ metadata: inspection.metadata,
204
+ blocks,
205
+ images: byId,
206
+ embeddedPlatforms: platforms,
207
+ });
208
+ const textImageCount = resolvedImages.filter((image) => image.kind === "text").length;
209
+ const mixedImageCount = resolvedImages.filter((image) => image.kind === "mixed").length;
210
+ const visualImageCount = resolvedImages.filter((image) => image.kind === "visual").length;
211
+ const annotationCount = resolvedImages.filter((image) =>
212
+ image.method === "agent" || image.method === "manual").length;
213
+ const tesseractCount = resolvedImages.filter((image) => image.method === "tesseract").length;
214
+ const ocr: PdfCaptureManifest["extraction"]["ocr"] = annotationCount > 0 && tesseractCount > 0
215
+ ? "mixed"
216
+ : annotationCount > 0
217
+ ? "annotations"
218
+ : tesseractCount > 0
219
+ ? "tesseract"
220
+ : "unavailable";
221
+ const finalWarnings = [...new Set(warnings)];
222
+ const annotationsJson = annotations.values.length === 0
223
+ ? null
224
+ : `${JSON.stringify(annotations.values, null, 2)}\n`;
225
+ const annotationsManifest: PdfCaptureManifest["annotations"] = annotationsJson === null
226
+ ? null
227
+ : {
228
+ path: PDF_CAPTURE_ANNOTATIONS_FILENAME,
229
+ count: annotations.values.length,
230
+ bytes: Buffer.byteLength(annotationsJson),
231
+ sha256: createHash("sha256").update(annotationsJson).digest("hex"),
232
+ };
233
+ const manifest: PdfCaptureManifest = {
234
+ schemaVersion: PDF_CAPTURE_MANIFEST_SCHEMA_VERSION,
235
+ kind: "pdf",
236
+ capturedAt,
237
+ status,
238
+ source: {
239
+ originalFilename: sanitizeTerminalText(inspection.originalFilename).slice(0, 4_096),
240
+ path: PDF_CAPTURE_SOURCE_FILENAME,
241
+ mimeType: "application/pdf",
242
+ bytes: inspection.sourceBytes,
243
+ sha256: inspection.sourceSha256,
244
+ ...(options.remoteSource === undefined
245
+ ? {}
246
+ : {
247
+ requestedUrl: options.remoteSource.requestedUrl,
248
+ finalUrl: options.remoteSource.finalUrl,
249
+ }),
250
+ },
251
+ document: {
252
+ ...inspection.metadata,
253
+ processedPages: inspection.processedPages,
254
+ },
255
+ extraction: {
256
+ layout: "pdftohtml-xml",
257
+ popplerVersion: inspection.popplerVersion,
258
+ ocr,
259
+ headingCount: built.headingCount,
260
+ textBlockCount: built.textBlockCount,
261
+ imageCount: resolvedImages.length,
262
+ textImageCount,
263
+ mixedImageCount,
264
+ visualImageCount,
265
+ },
266
+ images: resolvedImages.map(manifestImage),
267
+ annotations: annotationsManifest,
268
+ embeddedPlatforms: platforms,
269
+ warnings: finalWarnings,
270
+ };
271
+ const outputDirectory = persistPdfCapture({
272
+ outputBase: options.outputBase,
273
+ slug,
274
+ force: options.force ?? false,
275
+ sourcePath: inspection.inputPath,
276
+ markdown: built.markdown,
277
+ manifest,
278
+ images,
279
+ ...(annotationsJson === null ? {} : { annotationsJson }),
280
+ });
281
+ return {
282
+ status,
283
+ slug,
284
+ outputDirectory,
285
+ markdownPath: join(outputDirectory, pdfMarkdownFilename(slug)),
286
+ sourcePath: join(outputDirectory, PDF_CAPTURE_SOURCE_FILENAME),
287
+ wordCount: countWords(built.markdown),
288
+ pageCount: inspection.metadata.pageCount,
289
+ processedPages: inspection.processedPages,
290
+ imageCount: images.length,
291
+ warnings: finalWarnings,
292
+ markdown: built.markdown,
293
+ manifest,
294
+ };
295
+ } finally {
296
+ if (!callerOwnedWorkspace) rmSync(workspaceDirectory, { recursive: true, force: true });
297
+ }
298
+ }
package/src/pdf/cli.ts ADDED
@@ -0,0 +1,159 @@
1
+ import { lstatSync, readFileSync } from "node:fs";
2
+ import { resolve } from "node:path";
3
+
4
+ import { redactSensitiveText } from "../clip/persist.js";
5
+ import { sanitizeTerminalLine, sanitizeTerminalText } from "../clip/terminal.js";
6
+ import { parsePdfArguments, pdfUsage } from "./args.js";
7
+ import { runPdfCapture } from "./capture.js";
8
+ import { parsePdfImageInterpretations } from "./layout.js";
9
+ import { preparePdfSource, type PreparedPdfSource } from "./source.js";
10
+ import type {
11
+ PdfCaptureDependencies,
12
+ PdfCaptureOptions,
13
+ PdfCaptureOutcome,
14
+ PdfImageInterpretation,
15
+ } from "./model.js";
16
+
17
+ export type PdfCliOutput = {
18
+ readonly stdout: (value: string) => void;
19
+ readonly stderr: (value: string) => void;
20
+ };
21
+
22
+ const defaultOutput: PdfCliOutput = {
23
+ stdout: (value) => process.stdout.write(value),
24
+ stderr: (value) => process.stderr.write(value),
25
+ };
26
+
27
+ export type PdfCliDependencies = {
28
+ readonly runPdfCapture?: typeof runPdfCapture;
29
+ readonly captureDependencies?: PdfCaptureDependencies;
30
+ readonly readInterpretations?: (path: string) => readonly PdfImageInterpretation[];
31
+ readonly preparePdfSource?: typeof preparePdfSource;
32
+ };
33
+
34
+ function safe(value: string): string {
35
+ return sanitizeTerminalLine(redactSensitiveText(value));
36
+ }
37
+
38
+ function terminalSafeJson(value: unknown): string {
39
+ return `${JSON.stringify(
40
+ value,
41
+ (_key, candidate: unknown) => typeof candidate === "string"
42
+ ? sanitizeTerminalText(redactSensitiveText(candidate))
43
+ : candidate,
44
+ 2,
45
+ )}\n`;
46
+ }
47
+
48
+ function readInterpretations(path: string): readonly PdfImageInterpretation[] {
49
+ const absolute = resolve(path);
50
+ const stats = lstatSync(absolute);
51
+ if (!stats.isFile() || stats.isSymbolicLink()) {
52
+ throw new Error("PDF image annotations must be a regular JSON file");
53
+ }
54
+ if (stats.size > 16 * 1024 * 1024) {
55
+ throw new Error("PDF image annotations exceed the 16MB limit");
56
+ }
57
+ let value: unknown;
58
+ try {
59
+ value = JSON.parse(readFileSync(absolute, "utf8")) as unknown;
60
+ } catch {
61
+ throw new Error("PDF image annotations are not valid JSON");
62
+ }
63
+ return parsePdfImageInterpretations(value);
64
+ }
65
+
66
+ export function pdfCaptureSummary(outcome: PdfCaptureOutcome): Record<string, unknown> {
67
+ return {
68
+ ok: true,
69
+ status: outcome.status,
70
+ slug: outcome.slug,
71
+ outputDirectory: outcome.outputDirectory,
72
+ markdownPath: outcome.markdownPath,
73
+ sourcePath: outcome.sourcePath,
74
+ wordCount: outcome.wordCount,
75
+ pageCount: outcome.pageCount,
76
+ processedPages: outcome.processedPages,
77
+ imageCount: outcome.imageCount,
78
+ warnings: outcome.warnings,
79
+ manifest: outcome.manifest,
80
+ };
81
+ }
82
+
83
+ /** Delegated CLI entry point for the root `kb pdf` command. */
84
+ export async function main(
85
+ rawArguments: readonly string[] = process.argv.slice(2),
86
+ environment: Readonly<Record<string, string | undefined>> = process.env,
87
+ output: PdfCliOutput = defaultOutput,
88
+ dependencies: PdfCliDependencies = {},
89
+ ): Promise<number> {
90
+ const parsed = parsePdfArguments(rawArguments, environment);
91
+ if (!parsed.ok) {
92
+ output.stderr(`error: ${safe(parsed.message)}\n\n${sanitizeTerminalText(pdfUsage)}`);
93
+ return 2;
94
+ }
95
+ const arguments_ = parsed.value;
96
+ if (arguments_.command === "help") {
97
+ output.stdout(sanitizeTerminalText(pdfUsage));
98
+ return 0;
99
+ }
100
+ if (!arguments_.quiet && !arguments_.json) {
101
+ output.stderr(`Saving PDF ${safe(arguments_.input)} ...\n`);
102
+ }
103
+ let preparedSource: PreparedPdfSource | null = null;
104
+ try {
105
+ const interpretations = arguments_.interpretationsPath === undefined
106
+ ? undefined
107
+ : (dependencies.readInterpretations ?? readInterpretations)(arguments_.interpretationsPath);
108
+ preparedSource = await (dependencies.preparePdfSource ?? preparePdfSource)(
109
+ arguments_.input,
110
+ {
111
+ ...(arguments_.timeoutMs === undefined ? {} : { timeoutMs: arguments_.timeoutMs }),
112
+ ...(arguments_.maxPdfBytes === undefined ? {} : { maxPdfBytes: arguments_.maxPdfBytes }),
113
+ },
114
+ );
115
+ const options: PdfCaptureOptions = {
116
+ inputPath: preparedSource.inputPath,
117
+ outputBase: arguments_.outputBase,
118
+ ...(preparedSource.remoteSource === undefined ? {} : { remoteSource: preparedSource.remoteSource }),
119
+ ...(arguments_.slug === undefined ? {} : { slug: arguments_.slug }),
120
+ ...(interpretations === undefined ? {} : { interpretations }),
121
+ force: arguments_.force,
122
+ ...(arguments_.timeoutMs === undefined ? {} : { timeoutMs: arguments_.timeoutMs }),
123
+ ...(arguments_.maxPdfBytes === undefined ? {} : { maxPdfBytes: arguments_.maxPdfBytes }),
124
+ ...(arguments_.maxPages === undefined ? {} : { maxPages: arguments_.maxPages }),
125
+ ...(arguments_.maxImages === undefined ? {} : { maxImages: arguments_.maxImages }),
126
+ ...(arguments_.maxAssetBytes === undefined ? {} : { maxAssetBytes: arguments_.maxAssetBytes }),
127
+ ...(arguments_.maxTotalAssetBytes === undefined
128
+ ? {}
129
+ : { maxTotalAssetBytes: arguments_.maxTotalAssetBytes }),
130
+ };
131
+ const outcome = await (dependencies.runPdfCapture ?? runPdfCapture)(
132
+ options,
133
+ dependencies.captureDependencies,
134
+ );
135
+ if (arguments_.json) output.stdout(terminalSafeJson(pdfCaptureSummary(outcome)));
136
+ else {
137
+ output.stdout(`Done: ${safe(outcome.markdownPath)}\n`);
138
+ const pages = outcome.processedPages === outcome.pageCount
139
+ ? `${outcome.pageCount} pages`
140
+ : `${outcome.processedPages} of ${outcome.pageCount} pages processed`;
141
+ output.stdout(
142
+ `Status: ${outcome.status}; ${pages}; ${outcome.wordCount} words; ${outcome.imageCount} images.\n`,
143
+ );
144
+ }
145
+ if (!arguments_.quiet && !arguments_.json) {
146
+ for (const warning of outcome.warnings) output.stderr(`warning: ${safe(warning)}\n`);
147
+ }
148
+ return 0;
149
+ } catch (error) {
150
+ const message = safe(error instanceof Error ? error.message : String(error));
151
+ if (arguments_.json) output.stdout(terminalSafeJson({ ok: false, error: message }));
152
+ else output.stderr(`error: ${message}\n`);
153
+ return 1;
154
+ } finally {
155
+ preparedSource?.dispose();
156
+ }
157
+ }
158
+
159
+ if (import.meta.main) process.exitCode = await main();