abb-opencode-local-rag 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (233) hide show
  1. package/LICENSE +21 -0
  2. package/README.de.md +416 -0
  3. package/README.es.md +416 -0
  4. package/README.fr.md +416 -0
  5. package/README.md +491 -0
  6. package/README.pt-BR.md +416 -0
  7. package/README.zh-CN.md +416 -0
  8. package/dist/bin/install-skills.d.ts +20 -0
  9. package/dist/bin/install-skills.d.ts.map +1 -0
  10. package/dist/bin/install-skills.js +195 -0
  11. package/dist/bin/install-skills.js.map +1 -0
  12. package/dist/chunker/index.d.ts +24 -0
  13. package/dist/chunker/index.d.ts.map +1 -0
  14. package/dist/chunker/index.js +2 -0
  15. package/dist/chunker/index.js.map +1 -0
  16. package/dist/chunker/semantic-chunker.d.ts +97 -0
  17. package/dist/chunker/semantic-chunker.d.ts.map +1 -0
  18. package/dist/chunker/semantic-chunker.js +294 -0
  19. package/dist/chunker/semantic-chunker.js.map +1 -0
  20. package/dist/chunker/sentence-splitter.d.ts +28 -0
  21. package/dist/chunker/sentence-splitter.d.ts.map +1 -0
  22. package/dist/chunker/sentence-splitter.js +219 -0
  23. package/dist/chunker/sentence-splitter.js.map +1 -0
  24. package/dist/cli/common.d.ts +65 -0
  25. package/dist/cli/common.d.ts.map +1 -0
  26. package/dist/cli/common.js +138 -0
  27. package/dist/cli/common.js.map +1 -0
  28. package/dist/cli/delete.d.ts +8 -0
  29. package/dist/cli/delete.d.ts.map +1 -0
  30. package/dist/cli/delete.js +173 -0
  31. package/dist/cli/delete.js.map +1 -0
  32. package/dist/cli/file-collection.d.ts +2 -0
  33. package/dist/cli/file-collection.d.ts.map +1 -0
  34. package/dist/cli/file-collection.js +53 -0
  35. package/dist/cli/file-collection.js.map +1 -0
  36. package/dist/cli/ingest.d.ts +100 -0
  37. package/dist/cli/ingest.d.ts.map +1 -0
  38. package/dist/cli/ingest.js +363 -0
  39. package/dist/cli/ingest.js.map +1 -0
  40. package/dist/cli/list.d.ts +35 -0
  41. package/dist/cli/list.d.ts.map +1 -0
  42. package/dist/cli/list.js +210 -0
  43. package/dist/cli/list.js.map +1 -0
  44. package/dist/cli/options.d.ts +100 -0
  45. package/dist/cli/options.d.ts.map +1 -0
  46. package/dist/cli/options.js +241 -0
  47. package/dist/cli/options.js.map +1 -0
  48. package/dist/cli/query.d.ts +24 -0
  49. package/dist/cli/query.d.ts.map +1 -0
  50. package/dist/cli/query.js +191 -0
  51. package/dist/cli/query.js.map +1 -0
  52. package/dist/cli/read-neighbors.d.ts +11 -0
  53. package/dist/cli/read-neighbors.d.ts.map +1 -0
  54. package/dist/cli/read-neighbors.js +224 -0
  55. package/dist/cli/read-neighbors.js.map +1 -0
  56. package/dist/cli/status.d.ts +8 -0
  57. package/dist/cli/status.d.ts.map +1 -0
  58. package/dist/cli/status.js +80 -0
  59. package/dist/cli/status.js.map +1 -0
  60. package/dist/cli/sync.d.ts +8 -0
  61. package/dist/cli/sync.d.ts.map +1 -0
  62. package/dist/cli/sync.js +244 -0
  63. package/dist/cli/sync.js.map +1 -0
  64. package/dist/cli-main.d.ts +12 -0
  65. package/dist/cli-main.d.ts.map +1 -0
  66. package/dist/cli-main.js +63 -0
  67. package/dist/cli-main.js.map +1 -0
  68. package/dist/embedder/index.d.ts +85 -0
  69. package/dist/embedder/index.d.ts.map +1 -0
  70. package/dist/embedder/index.js +284 -0
  71. package/dist/embedder/index.js.map +1 -0
  72. package/dist/features/list.d.ts +37 -0
  73. package/dist/features/list.d.ts.map +1 -0
  74. package/dist/features/list.js +40 -0
  75. package/dist/features/list.js.map +1 -0
  76. package/dist/features/sync.d.ts +207 -0
  77. package/dist/features/sync.d.ts.map +1 -0
  78. package/dist/features/sync.js +380 -0
  79. package/dist/features/sync.js.map +1 -0
  80. package/dist/index.d.ts +3 -0
  81. package/dist/index.d.ts.map +1 -0
  82. package/dist/index.js +53 -0
  83. package/dist/index.js.map +1 -0
  84. package/dist/ingest/compute.d.ts +86 -0
  85. package/dist/ingest/compute.d.ts.map +1 -0
  86. package/dist/ingest/compute.js +177 -0
  87. package/dist/ingest/compute.js.map +1 -0
  88. package/dist/ingest/file.d.ts +27 -0
  89. package/dist/ingest/file.d.ts.map +1 -0
  90. package/dist/ingest/file.js +67 -0
  91. package/dist/ingest/file.js.map +1 -0
  92. package/dist/ingest/visual.d.ts +45 -0
  93. package/dist/ingest/visual.d.ts.map +1 -0
  94. package/dist/ingest/visual.js +234 -0
  95. package/dist/ingest/visual.js.map +1 -0
  96. package/dist/parser/docx-parser.d.ts +12 -0
  97. package/dist/parser/docx-parser.d.ts.map +1 -0
  98. package/dist/parser/docx-parser.js +328 -0
  99. package/dist/parser/docx-parser.js.map +1 -0
  100. package/dist/parser/html-parser.d.ts +18 -0
  101. package/dist/parser/html-parser.d.ts.map +1 -0
  102. package/dist/parser/html-parser.js +102 -0
  103. package/dist/parser/html-parser.js.map +1 -0
  104. package/dist/parser/index.d.ts +214 -0
  105. package/dist/parser/index.d.ts.map +1 -0
  106. package/dist/parser/index.js +454 -0
  107. package/dist/parser/index.js.map +1 -0
  108. package/dist/parser/pdf-extract.d.ts +81 -0
  109. package/dist/parser/pdf-extract.d.ts.map +1 -0
  110. package/dist/parser/pdf-extract.js +112 -0
  111. package/dist/parser/pdf-extract.js.map +1 -0
  112. package/dist/parser/pdf-filter.d.ts +117 -0
  113. package/dist/parser/pdf-filter.d.ts.map +1 -0
  114. package/dist/parser/pdf-filter.js +528 -0
  115. package/dist/parser/pdf-filter.js.map +1 -0
  116. package/dist/parser/title-extractor.d.ts +69 -0
  117. package/dist/parser/title-extractor.d.ts.map +1 -0
  118. package/dist/parser/title-extractor.js +145 -0
  119. package/dist/parser/title-extractor.js.map +1 -0
  120. package/dist/pdf-visual/captioner.d.ts +16 -0
  121. package/dist/pdf-visual/captioner.d.ts.map +1 -0
  122. package/dist/pdf-visual/captioner.js +63 -0
  123. package/dist/pdf-visual/captioner.js.map +1 -0
  124. package/dist/pdf-visual/captioners/fast.d.ts +7 -0
  125. package/dist/pdf-visual/captioners/fast.d.ts.map +1 -0
  126. package/dist/pdf-visual/captioners/fast.js +103 -0
  127. package/dist/pdf-visual/captioners/fast.js.map +1 -0
  128. package/dist/pdf-visual/captioners/quality.d.ts +7 -0
  129. package/dist/pdf-visual/captioners/quality.d.ts.map +1 -0
  130. package/dist/pdf-visual/captioners/quality.js +127 -0
  131. package/dist/pdf-visual/captioners/quality.js.map +1 -0
  132. package/dist/pdf-visual/captioners/shared.d.ts +44 -0
  133. package/dist/pdf-visual/captioners/shared.d.ts.map +1 -0
  134. package/dist/pdf-visual/captioners/shared.js +104 -0
  135. package/dist/pdf-visual/captioners/shared.js.map +1 -0
  136. package/dist/pdf-visual/detector.d.ts +9 -0
  137. package/dist/pdf-visual/detector.d.ts.map +1 -0
  138. package/dist/pdf-visual/detector.js +234 -0
  139. package/dist/pdf-visual/detector.js.map +1 -0
  140. package/dist/pdf-visual/index.d.ts +13 -0
  141. package/dist/pdf-visual/index.d.ts.map +1 -0
  142. package/dist/pdf-visual/index.js +45 -0
  143. package/dist/pdf-visual/index.js.map +1 -0
  144. package/dist/pdf-visual/renderer.d.ts +9 -0
  145. package/dist/pdf-visual/renderer.d.ts.map +1 -0
  146. package/dist/pdf-visual/renderer.js +177 -0
  147. package/dist/pdf-visual/renderer.js.map +1 -0
  148. package/dist/pdf-visual/types.d.ts +62 -0
  149. package/dist/pdf-visual/types.d.ts.map +1 -0
  150. package/dist/pdf-visual/types.js +32 -0
  151. package/dist/pdf-visual/types.js.map +1 -0
  152. package/dist/server/error-utils.d.ts +79 -0
  153. package/dist/server/error-utils.d.ts.map +1 -0
  154. package/dist/server/error-utils.js +148 -0
  155. package/dist/server/error-utils.js.map +1 -0
  156. package/dist/server/index.d.ts +258 -0
  157. package/dist/server/index.d.ts.map +1 -0
  158. package/dist/server/index.js +1104 -0
  159. package/dist/server/index.js.map +1 -0
  160. package/dist/server/list-scanner.d.ts +52 -0
  161. package/dist/server/list-scanner.d.ts.map +1 -0
  162. package/dist/server/list-scanner.js +72 -0
  163. package/dist/server/list-scanner.js.map +1 -0
  164. package/dist/server/tool-definitions.d.ts +8 -0
  165. package/dist/server/tool-definitions.d.ts.map +1 -0
  166. package/dist/server/tool-definitions.js +181 -0
  167. package/dist/server/tool-definitions.js.map +1 -0
  168. package/dist/server/tool-input.d.ts +37 -0
  169. package/dist/server/tool-input.d.ts.map +1 -0
  170. package/dist/server/tool-input.js +216 -0
  171. package/dist/server/tool-input.js.map +1 -0
  172. package/dist/server/types.d.ts +331 -0
  173. package/dist/server/types.d.ts.map +1 -0
  174. package/dist/server/types.js +3 -0
  175. package/dist/server/types.js.map +1 -0
  176. package/dist/server-main.d.ts +46 -0
  177. package/dist/server-main.d.ts.map +1 -0
  178. package/dist/server-main.js +242 -0
  179. package/dist/server-main.js.map +1 -0
  180. package/dist/utils/base-dirs.d.ts +212 -0
  181. package/dist/utils/base-dirs.d.ts.map +1 -0
  182. package/dist/utils/base-dirs.js +422 -0
  183. package/dist/utils/base-dirs.js.map +1 -0
  184. package/dist/utils/errors.d.ts +24 -0
  185. package/dist/utils/errors.d.ts.map +1 -0
  186. package/dist/utils/errors.js +53 -0
  187. package/dist/utils/errors.js.map +1 -0
  188. package/dist/utils/limits.d.ts +26 -0
  189. package/dist/utils/limits.d.ts.map +1 -0
  190. package/dist/utils/limits.js +28 -0
  191. package/dist/utils/limits.js.map +1 -0
  192. package/dist/utils/list-sources.d.ts +47 -0
  193. package/dist/utils/list-sources.d.ts.map +1 -0
  194. package/dist/utils/list-sources.js +50 -0
  195. package/dist/utils/list-sources.js.map +1 -0
  196. package/dist/utils/raw-data-utils.d.ts +131 -0
  197. package/dist/utils/raw-data-utils.d.ts.map +1 -0
  198. package/dist/utils/raw-data-utils.js +255 -0
  199. package/dist/utils/raw-data-utils.js.map +1 -0
  200. package/dist/utils/scan.d.ts +126 -0
  201. package/dist/utils/scan.d.ts.map +1 -0
  202. package/dist/utils/scan.js +221 -0
  203. package/dist/utils/scan.js.map +1 -0
  204. package/dist/utils/scope-match.d.ts +43 -0
  205. package/dist/utils/scope-match.d.ts.map +1 -0
  206. package/dist/utils/scope-match.js +87 -0
  207. package/dist/utils/scope-match.js.map +1 -0
  208. package/dist/utils/sensitive-path.d.ts +23 -0
  209. package/dist/utils/sensitive-path.d.ts.map +1 -0
  210. package/dist/utils/sensitive-path.js +91 -0
  211. package/dist/utils/sensitive-path.js.map +1 -0
  212. package/dist/utils/sync-path-key.d.ts +20 -0
  213. package/dist/utils/sync-path-key.d.ts.map +1 -0
  214. package/dist/utils/sync-path-key.js +33 -0
  215. package/dist/utils/sync-path-key.js.map +1 -0
  216. package/dist/vectordb/index.d.ts +168 -0
  217. package/dist/vectordb/index.d.ts.map +1 -0
  218. package/dist/vectordb/index.js +619 -0
  219. package/dist/vectordb/index.js.map +1 -0
  220. package/dist/vectordb/search-filters.d.ts +39 -0
  221. package/dist/vectordb/search-filters.d.ts.map +1 -0
  222. package/dist/vectordb/search-filters.js +136 -0
  223. package/dist/vectordb/search-filters.js.map +1 -0
  224. package/dist/vectordb/types.d.ts +196 -0
  225. package/dist/vectordb/types.d.ts.map +1 -0
  226. package/dist/vectordb/types.js +224 -0
  227. package/dist/vectordb/types.js.map +1 -0
  228. package/package.json +105 -0
  229. package/skills/mcp-local-rag/SKILL.md +308 -0
  230. package/skills/mcp-local-rag/references/cli-reference.md +175 -0
  231. package/skills/mcp-local-rag/references/html-ingestion.md +78 -0
  232. package/skills/mcp-local-rag/references/query-optimization.md +57 -0
  233. package/skills/mcp-local-rag/references/result-refinement.md +56 -0
@@ -0,0 +1,454 @@
1
+ // DocumentParser implementation with PDF/DOCX/TXT/MD support
2
+ import { statSync } from 'node:fs';
3
+ import { lstat, readFile, realpath } from 'node:fs/promises';
4
+ import { basename, extname, isAbsolute, resolve } from 'node:path';
5
+ import { JSDOM } from 'jsdom';
6
+ import mammoth from 'mammoth';
7
+ import { SemanticChunker } from '../chunker/index.js';
8
+ import { withTrailingSeparator } from '../utils/base-dirs.js';
9
+ import { AppError, isAppError } from '../utils/errors.js';
10
+ import { convertDocxDocumentToText, extractDocxCoreTitle } from './docx-parser.js';
11
+ import { extractPdfPages } from './pdf-extract.js';
12
+ import { extractDocxTitle, extractMarkdownTitle, extractPdfTitle, extractTxtTitle, } from './title-extractor.js';
13
+ // ============================================
14
+ // Supported Extensions
15
+ // ============================================
16
+ /**
17
+ * File extensions supported by the parser module (parseFile + parsePdf).
18
+ * Exported so other modules (e.g. list_files) stay in sync automatically
19
+ * when new formats are added here.
20
+ */
21
+ export const SUPPORTED_EXTENSIONS = new Set(['.pdf', '.docx', '.txt', '.md']);
22
+ async function resolvePdfTitle(filePath, pages, metadataTitle, page1FontHint, embedder) {
23
+ const fileName = basename(filePath);
24
+ let firstPageChunkText;
25
+ try {
26
+ const filteredPage1 = pages[0]?.text;
27
+ if (filteredPage1 && filteredPage1.trim().length > 0) {
28
+ const page1Chunks = await new SemanticChunker().chunkText(filteredPage1, embedder);
29
+ firstPageChunkText = page1Chunks[0]?.text;
30
+ }
31
+ }
32
+ catch (titleError) {
33
+ if (isAppError(titleError))
34
+ throw titleError;
35
+ console.error(`Title extraction failed, falling back to filename: ${titleError}`);
36
+ }
37
+ return extractPdfTitle(metadataTitle, firstPageChunkText, fileName, page1FontHint).title;
38
+ }
39
+ /**
40
+ * Validation error (equivalent to 400)
41
+ */
42
+ export class ValidationError extends AppError {
43
+ constructor(message, cause) {
44
+ super(message, 'parser', 'validation', cause);
45
+ this.name = 'ValidationError';
46
+ }
47
+ }
48
+ /**
49
+ * File operation error (equivalent to 500)
50
+ */
51
+ export class FileOperationError extends AppError {
52
+ constructor(message, cause) {
53
+ super(message, 'parser', 'io', cause);
54
+ this.name = 'FileOperationError';
55
+ }
56
+ }
57
+ // ============================================
58
+ // DocumentParser Class
59
+ // ============================================
60
+ /**
61
+ * Document parser class (PDF/DOCX/TXT/MD support)
62
+ *
63
+ * Responsibilities:
64
+ * - File path validation (path traversal prevention)
65
+ * - File size validation (100MB limit)
66
+ * - Parse 4 formats (PDF/DOCX/TXT/MD)
67
+ */
68
+ export class DocumentParser {
69
+ config;
70
+ /** Raw allowed roots in input order (pre-realpath). Always non-empty. */
71
+ rawBaseDirs;
72
+ /**
73
+ * Lazily cached realpath-normalized allowed roots, each with a trailing
74
+ * path separator so the `startsWith` check is sibling-prefix safe (e.g.
75
+ * `/foo/bar/` must not match `/foo/barista/x.txt`). Order is preserved
76
+ * from `rawBaseDirs` so the legacy single-root rejection message keeps
77
+ * referencing the user-configured first root. Assumes the allowed roots
78
+ * are stable for the process lifetime.
79
+ */
80
+ resolvedBaseDirs = null;
81
+ constructor(config) {
82
+ this.config = config;
83
+ // Normalize the two accepted shapes into one internal raw-root list.
84
+ // The type system already rejects supplying both fields simultaneously,
85
+ // but defensively pick `baseDirs` first so a future relaxation does not
86
+ // accidentally fall back to the legacy single-root field.
87
+ //
88
+ // Empty `baseDirs` is accepted here so the parser can be constructed in
89
+ // the MCP server's degraded mode (configError present); `validateFilePath`
90
+ // fails closed in that case so no file is accepted while the empty root
91
+ // list stands. This is the only legitimate way to reach empty
92
+ // `rawBaseDirs`; production wiring always supplies a non-empty list when
93
+ // `configError` is absent.
94
+ if (config.baseDirs !== undefined) {
95
+ this.rawBaseDirs = config.baseDirs;
96
+ }
97
+ else {
98
+ this.rawBaseDirs = [config.baseDir];
99
+ }
100
+ }
101
+ /**
102
+ * File path validation (Absolute path requirement + Path traversal prevention).
103
+ *
104
+ * This is THE place realpath is used (with the base-dir resolver): the
105
+ * security/containment boundary. Following symlinks here makes prefix
106
+ * containment unforgeable. Stored/scanned/looked-up paths elsewhere use
107
+ * resolve() — see {@link BaseDirsConfig} for the path policy.
108
+ *
109
+ * Multi-root semantics: a file is accepted iff its realpath (or, for a
110
+ * non-symlink path that does not yet exist, its `resolve()`-normalized
111
+ * absolute path) is under ANY realpath-normalized allowed root using a
112
+ * trailing-separator prefix check. Broken symlinks are still rejected
113
+ * outright — the lstat-based detection mirrors the previous single-root
114
+ * behavior.
115
+ *
116
+ * Under a single allowed root the behavior is identical to the previous
117
+ * single-root implementation.
118
+ *
119
+ * @param filePath - File path to validate (must be absolute)
120
+ * @throws ValidationError - When path is not absolute or outside all allowed roots
121
+ */
122
+ async validateFilePath(filePath) {
123
+ // Fail-closed in degraded mode: when the parser was constructed with an
124
+ // empty allow-list (only legitimate when the MCP server is in degraded
125
+ // mode with a configError set), reject every path with a structured
126
+ // error rather than performing the realpath check against an empty
127
+ // surviving-roots set. Server-level `assertConfigOk` should have fired
128
+ // first; this is a defense-in-depth fallback for code paths that
129
+ // bypass that gate.
130
+ if (this.rawBaseDirs.length === 0) {
131
+ throw new ValidationError('No configured base directory: file access is disabled. Resolve the BASE_DIR / BASE_DIRS configuration error reported by the `status` tool before retrying.');
132
+ }
133
+ // Check if path is absolute (fast-fail without syscall)
134
+ if (!isAbsolute(filePath)) {
135
+ throw new ValidationError(`File path must be absolute path (received: ${filePath}). Please provide an absolute path within a configured base directory (BASE_DIR/BASE_DIRS/--base-dir).`);
136
+ }
137
+ // Lazily resolve and cache the real path of each allowed root (follows
138
+ // symlinks). Each entry gets a trailing separator so subsequent
139
+ // `startsWith` checks are sibling-prefix safe.
140
+ if (!this.resolvedBaseDirs) {
141
+ const resolvedList = [];
142
+ for (const raw of this.rawBaseDirs) {
143
+ const resolved = await realpath(resolve(raw));
144
+ resolvedList.push(withTrailingSeparator(resolved));
145
+ }
146
+ this.resolvedBaseDirs = resolvedList;
147
+ }
148
+ // Resolve the real path of the file (follows symlinks)
149
+ let resolvedPath;
150
+ try {
151
+ resolvedPath = await realpath(filePath);
152
+ }
153
+ catch (error) {
154
+ // realpath fails if path doesn't exist on filesystem.
155
+ // Distinguish broken symlinks from genuinely non-existent paths:
156
+ // - Broken symlink: lstat succeeds (symlink entry exists) -> reject
157
+ // - Non-existent path: lstat fails -> fall back to resolve() for validation
158
+ const isSymlink = await lstat(filePath)
159
+ .then((stats) => stats.isSymbolicLink())
160
+ .catch(() => false);
161
+ if (isSymlink) {
162
+ throw new ValidationError(`Cannot resolve file path: ${filePath}. The file may not exist or is a broken symlink.`, error);
163
+ }
164
+ // File doesn't exist at all - fall back to resolve() for path validation.
165
+ // Note: resolve() is string-based and cannot detect symlinked parent directories.
166
+ // This is acceptable because non-existent files will fail at subsequent readFile/statSync.
167
+ resolvedPath = resolve(filePath);
168
+ }
169
+ // Check if resolved path is within any allowed root.
170
+ const allowed = this.resolvedBaseDirs.some((root) => resolvedPath.startsWith(root));
171
+ if (!allowed) {
172
+ const rootsDisplay = this.resolvedBaseDirs.length === 1
173
+ ? this.resolvedBaseDirs[0]
174
+ : this.resolvedBaseDirs.join(', ');
175
+ throw new ValidationError(`File path must be within a configured base directory (BASE_DIR/BASE_DIRS/--base-dir). Allowed roots: ${rootsDisplay}. Received path outside all configured roots: ${filePath}`);
176
+ }
177
+ }
178
+ /**
179
+ * File size validation (100MB limit)
180
+ *
181
+ * @param filePath - File path to validate
182
+ * @throws ValidationError - When file size exceeds limit
183
+ * @throws FileOperationError - When file read fails
184
+ */
185
+ validateFileSize(filePath) {
186
+ try {
187
+ const stats = statSync(filePath);
188
+ if (stats.size > this.config.maxFileSize) {
189
+ throw new ValidationError(`File size exceeds limit: ${stats.size} > ${this.config.maxFileSize}`);
190
+ }
191
+ }
192
+ catch (error) {
193
+ if (error instanceof ValidationError) {
194
+ throw error;
195
+ }
196
+ // Missing file is an input error, not an I/O fault.
197
+ if (error.code === 'ENOENT') {
198
+ throw new ValidationError(`File not found: ${filePath}`);
199
+ }
200
+ throw new FileOperationError(`Failed to check file size: ${filePath}`, error);
201
+ }
202
+ }
203
+ /**
204
+ * File parsing (auto format detection)
205
+ *
206
+ * @param filePath - File path to parse
207
+ * @returns ParseResult with content and extracted title
208
+ * @throws ValidationError - Path traversal, size exceeded, unsupported format
209
+ * @throws FileOperationError - File read failed, parse failed
210
+ */
211
+ async parseFile(filePath, options = {}) {
212
+ // Validation
213
+ await this.validateFilePath(filePath);
214
+ this.validateFileSize(filePath);
215
+ // Format detection (PDF uses parsePdf directly)
216
+ const ext = extname(filePath).toLowerCase();
217
+ switch (ext) {
218
+ case '.docx':
219
+ return await this.parseDocx(filePath, options.images === true);
220
+ case '.txt':
221
+ return await this.parseTxt(filePath);
222
+ case '.md':
223
+ return await this.parseMd(filePath);
224
+ default:
225
+ throw new ValidationError(`Unsupported file format: ${ext}`);
226
+ }
227
+ }
228
+ /**
229
+ * PDF parsing with header/footer filtering
230
+ *
231
+ * Features:
232
+ * - Extracts text with position information (x, y, fontSize)
233
+ * - Semantic header/footer detection using embedding similarity
234
+ * - Uses hasEOL for proper line break handling
235
+ * - Extracts document title from PDF metadata and first page font heuristic
236
+ *
237
+ * @param filePath - PDF file path
238
+ * @param embedder - Embedder for semantic header/footer detection
239
+ * @returns ParseResult with content and extracted title
240
+ * @throws FileOperationError - File read failed, parse failed
241
+ */
242
+ async parsePdf(filePath, embedder) {
243
+ // Validation
244
+ await this.validateFilePath(filePath);
245
+ this.validateFileSize(filePath);
246
+ // Hold `doc` outside the try so the `finally` block can dispose it after
247
+ // either a successful return or an error from `extractPdfPages` / the
248
+ // post-processing steps. `doc` stays `undefined` if `openDocument` itself
249
+ // throws — in that case there is no handle to destroy.
250
+ let doc;
251
+ try {
252
+ const buffer = await readFile(filePath);
253
+ const mupdf = await import('mupdf');
254
+ doc = mupdf.Document.openDocument(buffer, 'application/pdf');
255
+ const { pages, metadataTitle, page1FontHint } = await extractPdfPages(doc, embedder, 'preserve-whitespace');
256
+ const text = pages
257
+ .map((p) => p.text)
258
+ .filter((t) => t.length > 0)
259
+ .join('\n\n');
260
+ const title = await resolvePdfTitle(filePath, pages, metadataTitle, page1FontHint, embedder);
261
+ console.error(`Parsed PDF: ${filePath} (${text.length} characters, ${pages.length} pages)`);
262
+ return { content: text, title };
263
+ }
264
+ catch (error) {
265
+ // A foreign domain error (e.g. `EmbeddingError` raised while the parser
266
+ // uses the embedder) keeps its identity — rethrow it unchanged instead
267
+ // of relabeling it as a PDF parse failure. The parser's own
268
+ // `ValidationError`/`FileOperationError` are also `AppError` and so
269
+ // rethrow as-is, preserving their identity. Only a genuine non-`AppError`
270
+ // IO/mupdf failure wraps as `FileOperationError` with its `.cause` set.
271
+ if (isAppError(error)) {
272
+ throw error;
273
+ }
274
+ throw new FileOperationError(`Failed to parse PDF: ${filePath}`, error);
275
+ }
276
+ finally {
277
+ // Release the native WASM handle exactly once per invocation, on both
278
+ // success and error paths.
279
+ doc?.destroy();
280
+ }
281
+ }
282
+ /**
283
+ * Per-page PDF parsing for the visual-enrichment path.
284
+ *
285
+ * Opens a mupdf `Document`, delegates per-page extraction to the shared
286
+ * `extractPdfPages` helper with the `'preserve-whitespace,preserve-images'`
287
+ * stext option string so mupdf emits `block.type === 'image'` blocks for
288
+ * the downstream visual-candidate detector.
289
+ *
290
+ * Returns the open `Document` handle alongside the per-page records and
291
+ * title-resolution materials so the caller can:
292
+ * - run the renderer (`page.toPixmap()`) on the same handle,
293
+ * - feed `metadataTitle` + `pages[0].page1FontHint` into `extractPdfTitle`
294
+ * after `buildChunksAndEmbeddings` returns.
295
+ *
296
+ * Disposal contract (asymmetric — read carefully):
297
+ * - SUCCESS path: this method returns the open `doc` handle. The caller
298
+ * owns disposal and MUST wrap the call site in
299
+ * `try { ... } finally { doc.destroy() }`.
300
+ * - ERROR path: when this method throws, `doc` has already been destroyed
301
+ * internally before the exception propagates (so the caller never
302
+ * receives a handle it would not know to clean up). Callers MUST NOT
303
+ * call `doc.destroy()` on an error from this method.
304
+ * Title resolution stays here so visual captions cannot change PDF metadata.
305
+ * Visual candidate selection remains `pdf-visual/detector`'s responsibility.
306
+ *
307
+ * @param filePath - PDF file path (validated against BASE_DIR and size limit)
308
+ * @param embedder - Embedder for semantic header/footer detection
309
+ * @returns Open mupdf `Document`, resolved title, and per-page text/layout records.
310
+ * @throws ValidationError - Path traversal, size exceeded
311
+ * @throws FileOperationError - File read or parse failed (after destroying `doc` internally)
312
+ */
313
+ async parsePdfPages(filePath, embedder) {
314
+ // Validation (mirrors parsePdf's entry-point contract so the visual path
315
+ // does not bypass BASE_DIR / size checks).
316
+ await this.validateFilePath(filePath);
317
+ this.validateFileSize(filePath);
318
+ // Open the doc and run per-page extraction. Success-path disposal of
319
+ // `doc` stays with the caller.
320
+ // For the error-path window between `openDocument` and the return below,
321
+ // destroy `doc` here before re-throwing so a failure in `extractPdfPages`
322
+ // (or any future pre-return step) does not leak the mupdf WASM handle.
323
+ let doc;
324
+ try {
325
+ const buffer = await readFile(filePath);
326
+ const mupdf = await import('mupdf');
327
+ doc = mupdf.Document.openDocument(buffer, 'application/pdf');
328
+ const extracted = await extractPdfPages(doc, embedder, 'preserve-whitespace,preserve-images');
329
+ const { pages: helperPages, metadataTitle, page1FontHint } = extracted;
330
+ const pages = helperPages.map((page) => ({
331
+ pageNum: page.pageNum,
332
+ text: page.text,
333
+ textFragments: page.textFragments,
334
+ stextJson: page.stextJson,
335
+ }));
336
+ const title = await resolvePdfTitle(filePath, helperPages, metadataTitle, page1FontHint, embedder);
337
+ console.error(`Parsed PDF pages: ${filePath} (${pages.length} pages; caller owns doc disposal)`);
338
+ return { doc, title, pages };
339
+ }
340
+ catch (error) {
341
+ // `doc` is undefined when `openDocument` itself threw — nothing to free.
342
+ // When it is defined, dispose before re-throwing (on BOTH the foreign and
343
+ // the genuine error paths) so the caller never receives the handle and
344
+ // cannot be expected to clean it up.
345
+ doc?.destroy();
346
+ // A foreign domain error (e.g. `EmbeddingError`) keeps its identity —
347
+ // rethrow it unchanged. Only a genuine non-`AppError` IO/mupdf failure
348
+ // wraps as `FileOperationError` with its `.cause` set.
349
+ if (isAppError(error)) {
350
+ throw error;
351
+ }
352
+ throw new FileOperationError(`Failed to parse PDF pages: ${filePath}`, error);
353
+ }
354
+ }
355
+ /**
356
+ * DOCX parsing (using mammoth)
357
+ *
358
+ * Uses DOCX package metadata plus one Mammoth HTML conversion for title and body extraction.
359
+ *
360
+ * @param filePath - DOCX file path
361
+ * @returns ParseResult with content and extracted title
362
+ * @throws FileOperationError - File read failed, parse failed
363
+ */
364
+ async parseDocx(filePath, includeImages) {
365
+ try {
366
+ const buffer = await readFile(filePath);
367
+ const capturedImages = new Map();
368
+ let nextImageIndex = 0;
369
+ let skippedImageCount = 0;
370
+ const htmlResult = includeImages
371
+ ? await mammoth.convertToHtml({ buffer }, {
372
+ convertImage: mammoth.images.imgElement(async (image) => {
373
+ const imageIndex = nextImageIndex++;
374
+ if (image.contentType !== 'image/png' && image.contentType !== 'image/jpeg') {
375
+ skippedImageCount += 1;
376
+ return { src: `data:${image.contentType};base64,` };
377
+ }
378
+ const bytes = await image.readAsBuffer().catch(() => null);
379
+ if (bytes === null) {
380
+ skippedImageCount += 1;
381
+ return { src: `data:${image.contentType};base64,` };
382
+ }
383
+ capturedImages.set(imageIndex, { mimeType: image.contentType, bytes });
384
+ return {
385
+ src: `data:${image.contentType};base64,`,
386
+ 'data-rag-image-index': String(imageIndex),
387
+ };
388
+ }),
389
+ })
390
+ : await mammoth.convertToHtml({ buffer });
391
+ const htmlDocument = new JSDOM(htmlResult.value).window.document;
392
+ const coreTitle = await extractDocxCoreTitle(buffer);
393
+ const body = convertDocxDocumentToText(htmlDocument);
394
+ const fileName = basename(filePath);
395
+ const titleResult = extractDocxTitle(htmlDocument, fileName, coreTitle);
396
+ console.error(`Parsed DOCX: ${filePath} (${body.content.length} characters)`);
397
+ if (skippedImageCount > 0) {
398
+ console.warn(`Skipped ${skippedImageCount} unsupported or unreadable DOCX image(s) in ${filePath}; only embedded PNG and JPEG are supported`);
399
+ }
400
+ const imageAnchors = (body.imageAnchors ?? []).flatMap((anchor) => {
401
+ const captured = capturedImages.get(anchor.imageIndex);
402
+ return captured ? [{ ...anchor, ...captured }] : [];
403
+ });
404
+ return {
405
+ content: body.content,
406
+ title: titleResult.title,
407
+ ...(body.atomicRanges.length === 0 ? {} : { atomicRanges: body.atomicRanges }),
408
+ ...(imageAnchors.length === 0 ? {} : { imageAnchors }),
409
+ };
410
+ }
411
+ catch (error) {
412
+ throw new FileOperationError(`Failed to parse DOCX: ${filePath}`, error);
413
+ }
414
+ }
415
+ /**
416
+ * TXT parsing (using fs.readFile)
417
+ *
418
+ * @param filePath - TXT file path
419
+ * @returns ParseResult with content and extracted title
420
+ * @throws FileOperationError - File read failed
421
+ */
422
+ async parseTxt(filePath) {
423
+ try {
424
+ const text = await readFile(filePath, 'utf-8');
425
+ const fileName = basename(filePath);
426
+ const titleResult = extractTxtTitle(text, fileName);
427
+ console.error(`Parsed TXT: ${filePath} (${text.length} characters)`);
428
+ return { content: text, title: titleResult.title };
429
+ }
430
+ catch (error) {
431
+ throw new FileOperationError(`Failed to parse TXT: ${filePath}`, error);
432
+ }
433
+ }
434
+ /**
435
+ * MD parsing (using fs.readFile)
436
+ *
437
+ * @param filePath - MD file path
438
+ * @returns ParseResult with content and extracted title
439
+ * @throws FileOperationError - File read failed
440
+ */
441
+ async parseMd(filePath) {
442
+ try {
443
+ const text = await readFile(filePath, 'utf-8');
444
+ const fileName = basename(filePath);
445
+ const titleResult = extractMarkdownTitle(text, fileName);
446
+ console.error(`Parsed MD: ${filePath} (${text.length} characters)`);
447
+ return { content: text, title: titleResult.title };
448
+ }
449
+ catch (error) {
450
+ throw new FileOperationError(`Failed to parse MD: ${filePath}`, error);
451
+ }
452
+ }
453
+ }
454
+ //# sourceMappingURL=index.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/parser/index.ts"],"names":[],"mappings":"AAAA,6DAA6D;AAE7D,OAAO,EAAE,QAAQ,EAAE,MAAM,SAAS,CAAA;AAClC,OAAO,EAAE,KAAK,EAAE,QAAQ,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAA;AAC5D,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAE,UAAU,EAAE,OAAO,EAAE,MAAM,WAAW,CAAA;AAClE,OAAO,EAAE,KAAK,EAAE,MAAM,OAAO,CAAA;AAC7B,OAAO,OAAO,MAAM,SAAS,CAAA;AAE7B,OAAO,EAAwB,eAAe,EAAE,MAAM,qBAAqB,CAAA;AAC3E,OAAO,EAAE,qBAAqB,EAAE,MAAM,uBAAuB,CAAA;AAC7D,OAAO,EAAE,QAAQ,EAAE,UAAU,EAAE,MAAM,oBAAoB,CAAA;AACzD,OAAO,EAAE,yBAAyB,EAAE,oBAAoB,EAAE,MAAM,kBAAkB,CAAA;AAClF,OAAO,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAA;AAElD,OAAO,EACL,gBAAgB,EAChB,oBAAoB,EACpB,eAAe,EACf,eAAe,GAChB,MAAM,sBAAsB,CAAA;AAE7B,+CAA+C;AAC/C,uBAAuB;AACvB,+CAA+C;AAE/C;;;;GAIG;AACH,MAAM,CAAC,MAAM,oBAAoB,GAAG,IAAI,GAAG,CAAC,CAAC,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,KAAK,CAAC,CAAC,CAAA;AA4B7E,KAAK,UAAU,eAAe,CAC5B,QAAgB,EAChB,KAAkC,EAClC,aAAiC,EACjC,aAA6D,EAC7D,QAA2B;IAE3B,MAAM,QAAQ,GAAG,QAAQ,CAAC,QAAQ,CAAC,CAAA;IACnC,IAAI,kBAAsC,CAAA;IAC1C,IAAI,CAAC;QACH,MAAM,aAAa,GAAG,KAAK,CAAC,CAAC,CAAC,EAAE,IAAI,CAAA;QACpC,IAAI,aAAa,IAAI,aAAa,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACrD,MAAM,WAAW,GAAG,MAAM,IAAI,eAAe,EAAE,CAAC,SAAS,CAAC,aAAa,EAAE,QAAQ,CAAC,CAAA;YAClF,kBAAkB,GAAG,WAAW,CAAC,CAAC,CAAC,EAAE,IAAI,CAAA;QAC3C,CAAC;IACH,CAAC;IAAC,OAAO,UAAU,EAAE,CAAC;QACpB,IAAI,UAAU,CAAC,UAAU,CAAC;YAAE,MAAM,UAAU,CAAA;QAC5C,OAAO,CAAC,KAAK,CAAC,sDAAsD,UAAU,EAAE,CAAC,CAAA;IACnF,CAAC;IACD,OAAO,eAAe,CAAC,aAAa,EAAE,kBAAkB,EAAE,QAAQ,EAAE,aAAa,CAAC,CAAC,KAAK,CAAA;AAC1F,CAAC;AAiCD;;GAEG;AACH,MAAM,OAAO,eAAgB,SAAQ,QAAQ;IAC3C,YAAY,OAAe,EAAE,KAAa;QACxC,KAAK,CAAC,OAAO,EAAE,QAAQ,EAAE,YAAY,EAAE,KAAK,CAAC,CAAA;QAC7C,IAAI,CAAC,IAAI,GAAG,iBAAiB,CAAA;IAC/B,CAAC;CACF;AAED;;GAEG;AACH,MAAM,OAAO,kBAAmB,SAAQ,QAAQ;IAC9C,YAAY,OAAe,EAAE,KAAa;QACxC,KAAK,CAAC,OAAO,EAAE,QAAQ,EAAE,IAAI,EAAE,KAAK,CAAC,CAAA;QACrC,IAAI,CAAC,IAAI,GAAG,oBAAoB,CAAA;IAClC,CAAC;CACF;AAED,+CAA+C;AAC/C,uBAAuB;AACvB,+CAA+C;AAE/C;;;;;;;GAOG;AACH,MAAM,OAAO,cAAc;IACR,MAAM,CAAc;IACrC,yEAAyE;IACxD,WAAW,CAAmB;IAC/C;;;;;;;OAOG;IACK,gBAAgB,GAAoB,IAAI,CAAA;IAEhD,YAAY,MAAoB;QAC9B,IAAI,CAAC,MAAM,GAAG,MAAM,CAAA;QACpB,qEAAqE;QACrE,wEAAwE;QACxE,wEAAwE;QACxE,0DAA0D;QAC1D,EAAE;QACF,wEAAwE;QACxE,2EAA2E;QAC3E,wEAAwE;QACxE,8DAA8D;QAC9D,yEAAyE;QACzE,2BAA2B;QAC3B,IAAI,MAAM,CAAC,QAAQ,KAAK,SAAS,EAAE,CAAC;YAClC,IAAI,CAAC,WAAW,GAAG,MAAM,CAAC,QAAQ,CAAA;QACpC,CAAC;aAAM,CAAC;YACN,IAAI,CAAC,WAAW,GAAG,CAAC,MAAM,CAAC,OAAO,CAAC,CAAA;QACrC,CAAC;IACH,CAAC;IAED;;;;;;;;;;;;;;;;;;;;OAoBG;IACH,KAAK,CAAC,gBAAgB,CAAC,QAAgB;QACrC,wEAAwE;QACxE,uEAAuE;QACvE,oEAAoE;QACpE,mEAAmE;QACnE,uEAAuE;QACvE,iEAAiE;QACjE,oBAAoB;QACpB,IAAI,IAAI,CAAC,WAAW,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAClC,MAAM,IAAI,eAAe,CACvB,4JAA4J,CAC7J,CAAA;QACH,CAAC;QAED,wDAAwD;QACxD,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC,EAAE,CAAC;YAC1B,MAAM,IAAI,eAAe,CACvB,8CAA8C,QAAQ,wGAAwG,CAC/J,CAAA;QACH,CAAC;QAED,uEAAuE;QACvE,gEAAgE;QAChE,+CAA+C;QAC/C,IAAI,CAAC,IAAI,CAAC,gBAAgB,EAAE,CAAC;YAC3B,MAAM,YAAY,GAAa,EAAE,CAAA;YACjC,KAAK,MAAM,GAAG,IAAI,IAAI,CAAC,WAAW,EAAE,CAAC;gBACnC,MAAM,QAAQ,GAAG,MAAM,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAA;gBAC7C,YAAY,CAAC,IAAI,CAAC,qBAAqB,CAAC,QAAQ,CAAC,CAAC,CAAA;YACpD,CAAC;YACD,IAAI,CAAC,gBAAgB,GAAG,YAAY,CAAA;QACtC,CAAC;QAED,uDAAuD;QACvD,IAAI,YAAoB,CAAA;QACxB,IAAI,CAAC;YACH,YAAY,GAAG,MAAM,QAAQ,CAAC,QAAQ,CAAC,CAAA;QACzC,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,sDAAsD;YACtD,iEAAiE;YACjE,oEAAoE;YACpE,4EAA4E;YAC5E,MAAM,SAAS,GAAG,MAAM,KAAK,CAAC,QAAQ,CAAC;iBACpC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,cAAc,EAAE,CAAC;iBACvC,KAAK,CAAC,GAAG,EAAE,CAAC,KAAK,CAAC,CAAA;YAErB,IAAI,SAAS,EAAE,CAAC;gBACd,MAAM,IAAI,eAAe,CACvB,6BAA6B,QAAQ,kDAAkD,EACvF,KAAc,CACf,CAAA;YACH,CAAC;YAED,0EAA0E;YAC1E,kFAAkF;YAClF,2FAA2F;YAC3F,YAAY,GAAG,OAAO,CAAC,QAAQ,CAAC,CAAA;QAClC,CAAC;QAED,qDAAqD;QACrD,MAAM,OAAO,GAAG,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,YAAY,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,CAAA;QACnF,IAAI,CAAC,OAAO,EAAE,CAAC;YACb,MAAM,YAAY,GAChB,IAAI,CAAC,gBAAgB,CAAC,MAAM,KAAK,CAAC;gBAChC,CAAC,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC,CAAC;gBAC1B,CAAC,CAAC,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;YACtC,MAAM,IAAI,eAAe,CACvB,wGAAwG,YAAY,iDAAiD,QAAQ,EAAE,CAChL,CAAA;QACH,CAAC;IACH,CAAC;IAED;;;;;;OAMG;IACH,gBAAgB,CAAC,QAAgB;QAC/B,IAAI,CAAC;YACH,MAAM,KAAK,GAAG,QAAQ,CAAC,QAAQ,CAAC,CAAA;YAChC,IAAI,KAAK,CAAC,IAAI,GAAG,IAAI,CAAC,MAAM,CAAC,WAAW,EAAE,CAAC;gBACzC,MAAM,IAAI,eAAe,CACvB,4BAA4B,KAAK,CAAC,IAAI,MAAM,IAAI,CAAC,MAAM,CAAC,WAAW,EAAE,CACtE,CAAA;YACH,CAAC;QACH,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,KAAK,YAAY,eAAe,EAAE,CAAC;gBACrC,MAAM,KAAK,CAAA;YACb,CAAC;YACD,oDAAoD;YACpD,IAAK,KAA+B,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;gBACvD,MAAM,IAAI,eAAe,CAAC,mBAAmB,QAAQ,EAAE,CAAC,CAAA;YAC1D,CAAC;YACD,MAAM,IAAI,kBAAkB,CAAC,8BAA8B,QAAQ,EAAE,EAAE,KAAc,CAAC,CAAA;QACxF,CAAC;IACH,CAAC;IAED;;;;;;;OAOG;IACH,KAAK,CAAC,SAAS,CAAC,QAAgB,EAAE,OAAO,GAAqB,EAAE;QAC9D,aAAa;QACb,MAAM,IAAI,CAAC,gBAAgB,CAAC,QAAQ,CAAC,CAAA;QACrC,IAAI,CAAC,gBAAgB,CAAC,QAAQ,CAAC,CAAA;QAE/B,gDAAgD;QAChD,MAAM,GAAG,GAAG,OAAO,CAAC,QAAQ,CAAC,CAAC,WAAW,EAAE,CAAA;QAC3C,QAAQ,GAAG,EAAE,CAAC;YACZ,KAAK,OAAO;gBACV,OAAO,MAAM,IAAI,CAAC,SAAS,CAAC,QAAQ,EAAE,OAAO,CAAC,MAAM,KAAK,IAAI,CAAC,CAAA;YAChE,KAAK,MAAM;gBACT,OAAO,MAAM,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAA;YACtC,KAAK,KAAK;gBACR,OAAO,MAAM,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAA;YACrC;gBACE,MAAM,IAAI,eAAe,CAAC,4BAA4B,GAAG,EAAE,CAAC,CAAA;QAChE,CAAC;IACH,CAAC;IAED;;;;;;;;;;;;;OAaG;IACH,KAAK,CAAC,QAAQ,CAAC,QAAgB,EAAE,QAA2B;QAC1D,aAAa;QACb,MAAM,IAAI,CAAC,gBAAgB,CAAC,QAAQ,CAAC,CAAA;QACrC,IAAI,CAAC,gBAAgB,CAAC,QAAQ,CAAC,CAAA;QAE/B,yEAAyE;QACzE,sEAAsE;QACtE,0EAA0E;QAC1E,uDAAuD;QACvD,IAAI,GAA8B,CAAA;QAClC,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAC,QAAQ,CAAC,CAAA;YACvC,MAAM,KAAK,GAAG,MAAM,MAAM,CAAC,OAAO,CAAC,CAAA;YACnC,GAAG,GAAG,KAAK,CAAC,QAAQ,CAAC,YAAY,CAAC,MAAM,EAAE,iBAAiB,CAAkB,CAAA;YAE7E,MAAM,EAAE,KAAK,EAAE,aAAa,EAAE,aAAa,EAAE,GAAG,MAAM,eAAe,CACnE,GAAG,EACH,QAAQ,EACR,qBAAqB,CACtB,CAAA;YACD,MAAM,IAAI,GAAG,KAAK;iBACf,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;iBAClB,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC;iBAC3B,IAAI,CAAC,MAAM,CAAC,CAAA;YAEf,MAAM,KAAK,GAAG,MAAM,eAAe,CAAC,QAAQ,EAAE,KAAK,EAAE,aAAa,EAAE,aAAa,EAAE,QAAQ,CAAC,CAAA;YAE5F,OAAO,CAAC,KAAK,CAAC,eAAe,QAAQ,KAAK,IAAI,CAAC,MAAM,gBAAgB,KAAK,CAAC,MAAM,SAAS,CAAC,CAAA;YAE3F,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,CAAA;QACjC,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,wEAAwE;YACxE,uEAAuE;YACvE,4DAA4D;YAC5D,oEAAoE;YACpE,0EAA0E;YAC1E,wEAAwE;YACxE,IAAI,UAAU,CAAC,KAAK,CAAC,EAAE,CAAC;gBACtB,MAAM,KAAK,CAAA;YACb,CAAC;YACD,MAAM,IAAI,kBAAkB,CAAC,wBAAwB,QAAQ,EAAE,EAAE,KAAc,CAAC,CAAA;QAClF,CAAC;gBAAS,CAAC;YACT,sEAAsE;YACtE,2BAA2B;YAC3B,GAAG,EAAE,OAAO,EAAE,CAAA;QAChB,CAAC;IACH,CAAC;IAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OA8BG;IACH,KAAK,CAAC,aAAa,CACjB,QAAgB,EAChB,QAA2B;QAW3B,yEAAyE;QACzE,2CAA2C;QAC3C,MAAM,IAAI,CAAC,gBAAgB,CAAC,QAAQ,CAAC,CAAA;QACrC,IAAI,CAAC,gBAAgB,CAAC,QAAQ,CAAC,CAAA;QAE/B,qEAAqE;QACrE,+BAA+B;QAC/B,yEAAyE;QACzE,0EAA0E;QAC1E,uEAAuE;QACvE,IAAI,GAA8B,CAAA;QAClC,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAC,QAAQ,CAAC,CAAA;YACvC,MAAM,KAAK,GAAG,MAAM,MAAM,CAAC,OAAO,CAAC,CAAA;YACnC,GAAG,GAAG,KAAK,CAAC,QAAQ,CAAC,YAAY,CAAC,MAAM,EAAE,iBAAiB,CAAkB,CAAA;YAC7E,MAAM,SAAS,GAAG,MAAM,eAAe,CAAC,GAAG,EAAE,QAAQ,EAAE,qCAAqC,CAAC,CAAA;YAE7F,MAAM,EAAE,KAAK,EAAE,WAAW,EAAE,aAAa,EAAE,aAAa,EAAE,GAAG,SAAS,CAAA;YACtE,MAAM,KAAK,GAAG,WAAW,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;gBACvC,OAAO,EAAE,IAAI,CAAC,OAAO;gBACrB,IAAI,EAAE,IAAI,CAAC,IAAI;gBACf,aAAa,EAAE,IAAI,CAAC,aAAa;gBACjC,SAAS,EAAE,IAAI,CAAC,SAAS;aAC1B,CAAC,CAAC,CAAA;YACH,MAAM,KAAK,GAAG,MAAM,eAAe,CACjC,QAAQ,EACR,WAAW,EACX,aAAa,EACb,aAAa,EACb,QAAQ,CACT,CAAA;YAED,OAAO,CAAC,KAAK,CACX,qBAAqB,QAAQ,KAAK,KAAK,CAAC,MAAM,mCAAmC,CAClF,CAAA;YAED,OAAO,EAAE,GAAG,EAAE,KAAK,EAAE,KAAK,EAAE,CAAA;QAC9B,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,yEAAyE;YACzE,0EAA0E;YAC1E,uEAAuE;YACvE,qCAAqC;YACrC,GAAG,EAAE,OAAO,EAAE,CAAA;YACd,sEAAsE;YACtE,uEAAuE;YACvE,uDAAuD;YACvD,IAAI,UAAU,CAAC,KAAK,CAAC,EAAE,CAAC;gBACtB,MAAM,KAAK,CAAA;YACb,CAAC;YACD,MAAM,IAAI,kBAAkB,CAAC,8BAA8B,QAAQ,EAAE,EAAE,KAAc,CAAC,CAAA;QACxF,CAAC;IACH,CAAC;IAED;;;;;;;;OAQG;IACK,KAAK,CAAC,SAAS,CAAC,QAAgB,EAAE,aAAsB;QAC9D,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAC,QAAQ,CAAC,CAAA;YACvC,MAAM,cAAc,GAAG,IAAI,GAAG,EAG3B,CAAA;YACH,IAAI,cAAc,GAAG,CAAC,CAAA;YACtB,IAAI,iBAAiB,GAAG,CAAC,CAAA;YACzB,MAAM,UAAU,GAAG,aAAa;gBAC9B,CAAC,CAAC,MAAM,OAAO,CAAC,aAAa,CACzB,EAAE,MAAM,EAAE,EACV;oBACE,YAAY,EAAE,OAAO,CAAC,MAAM,CAAC,UAAU,CAAC,KAAK,EAAE,KAAK,EAAE,EAAE;wBACtD,MAAM,UAAU,GAAG,cAAc,EAAE,CAAA;wBACnC,IAAI,KAAK,CAAC,WAAW,KAAK,WAAW,IAAI,KAAK,CAAC,WAAW,KAAK,YAAY,EAAE,CAAC;4BAC5E,iBAAiB,IAAI,CAAC,CAAA;4BACtB,OAAO,EAAE,GAAG,EAAE,QAAQ,KAAK,CAAC,WAAW,UAAU,EAAE,CAAA;wBACrD,CAAC;wBACD,MAAM,KAAK,GAAG,MAAM,KAAK,CAAC,YAAY,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,CAAA;wBAC1D,IAAI,KAAK,KAAK,IAAI,EAAE,CAAC;4BACnB,iBAAiB,IAAI,CAAC,CAAA;4BACtB,OAAO,EAAE,GAAG,EAAE,QAAQ,KAAK,CAAC,WAAW,UAAU,EAAE,CAAA;wBACrD,CAAC;wBACD,cAAc,CAAC,GAAG,CAAC,UAAU,EAAE,EAAE,QAAQ,EAAE,KAAK,CAAC,WAAW,EAAE,KAAK,EAAE,CAAC,CAAA;wBACtE,OAAO;4BACL,GAAG,EAAE,QAAQ,KAAK,CAAC,WAAW,UAAU;4BACxC,sBAAsB,EAAE,MAAM,CAAC,UAAU,CAAC;yBAC3C,CAAA;oBACH,CAAC,CAAC;iBACH,CACF;gBACH,CAAC,CAAC,MAAM,OAAO,CAAC,aAAa,CAAC,EAAE,MAAM,EAAE,CAAC,CAAA;YAC3C,MAAM,YAAY,GAAG,IAAI,KAAK,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAA;YAChE,MAAM,SAAS,GAAG,MAAM,oBAAoB,CAAC,MAAM,CAAC,CAAA;YACpD,MAAM,IAAI,GAAG,yBAAyB,CAAC,YAAY,CAAC,CAAA;YACpD,MAAM,QAAQ,GAAG,QAAQ,CAAC,QAAQ,CAAC,CAAA;YACnC,MAAM,WAAW,GAAG,gBAAgB,CAAC,YAAY,EAAE,QAAQ,EAAE,SAAS,CAAC,CAAA;YAEvE,OAAO,CAAC,KAAK,CAAC,gBAAgB,QAAQ,KAAK,IAAI,CAAC,OAAO,CAAC,MAAM,cAAc,CAAC,CAAA;YAC7E,IAAI,iBAAiB,GAAG,CAAC,EAAE,CAAC;gBAC1B,OAAO,CAAC,IAAI,CACV,WAAW,iBAAiB,+CAA+C,QAAQ,4CAA4C,CAChI,CAAA;YACH,CAAC;YACD,MAAM,YAAY,GAAG,CAAC,IAAI,CAAC,YAAY,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,MAAM,EAAE,EAAE;gBAChE,MAAM,QAAQ,GAAG,cAAc,CAAC,GAAG,CAAC,MAAM,CAAC,UAAU,CAAC,CAAA;gBACtD,OAAO,QAAQ,CAAC,CAAC,CAAC,CAAC,EAAE,GAAG,MAAM,EAAE,GAAG,QAAQ,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,CAAA;YACrD,CAAC,CAAC,CAAA;YACF,OAAO;gBACL,OAAO,EAAE,IAAI,CAAC,OAAO;gBACrB,KAAK,EAAE,WAAW,CAAC,KAAK;gBACxB,GAAG,CAAC,IAAI,CAAC,YAAY,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,YAAY,EAAE,IAAI,CAAC,YAAY,EAAE,CAAC;gBAC9E,GAAG,CAAC,YAAY,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,YAAY,EAAE,CAAC;aACvD,CAAA;QACH,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,IAAI,kBAAkB,CAAC,yBAAyB,QAAQ,EAAE,EAAE,KAAc,CAAC,CAAA;QACnF,CAAC;IACH,CAAC;IAED;;;;;;OAMG;IACK,KAAK,CAAC,QAAQ,CAAC,QAAgB;QACrC,IAAI,CAAC;YACH,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAA;YAC9C,MAAM,QAAQ,GAAG,QAAQ,CAAC,QAAQ,CAAC,CAAA;YACnC,MAAM,WAAW,GAAG,eAAe,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAA;YACnD,OAAO,CAAC,KAAK,CAAC,eAAe,QAAQ,KAAK,IAAI,CAAC,MAAM,cAAc,CAAC,CAAA;YACpE,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,WAAW,CAAC,KAAK,EAAE,CAAA;QACpD,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,IAAI,kBAAkB,CAAC,wBAAwB,QAAQ,EAAE,EAAE,KAAc,CAAC,CAAA;QAClF,CAAC;IACH,CAAC;IAED;;;;;;OAMG;IACK,KAAK,CAAC,OAAO,CAAC,QAAgB;QACpC,IAAI,CAAC;YACH,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAA;YAC9C,MAAM,QAAQ,GAAG,QAAQ,CAAC,QAAQ,CAAC,CAAA;YACnC,MAAM,WAAW,GAAG,oBAAoB,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAA;YACxD,OAAO,CAAC,KAAK,CAAC,cAAc,QAAQ,KAAK,IAAI,CAAC,MAAM,cAAc,CAAC,CAAA;YACnE,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,WAAW,CAAC,KAAK,EAAE,CAAA;QACpD,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,IAAI,kBAAkB,CAAC,uBAAuB,QAAQ,EAAE,EAAE,KAAc,CAAC,CAAA;QACjF,CAAC;IACH,CAAC;CACF"}
@@ -0,0 +1,81 @@
1
+ import type { Document as MupdfDocument } from 'mupdf';
2
+ import { type EmbedderInterface, type FilteredTextFragment } from './pdf-filter.js';
3
+ interface StextBbox {
4
+ x: number;
5
+ y: number;
6
+ w: number;
7
+ h: number;
8
+ }
9
+ /**
10
+ * Shape of mupdf's structured-text JSON used by the per-page loop.
11
+ * Captured here so both the items-extraction step and the raw `stextJson`
12
+ * we return remain typed.
13
+ */
14
+ interface StextJson {
15
+ blocks: Array<{
16
+ type: string;
17
+ lines?: Array<{
18
+ text: string;
19
+ x: number;
20
+ y: number;
21
+ bbox?: StextBbox;
22
+ font: {
23
+ size: number;
24
+ name?: string;
25
+ weight?: string;
26
+ };
27
+ }>;
28
+ }>;
29
+ }
30
+ /**
31
+ * Per-page record produced by `extractPdfPages`. `text` is the page's text
32
+ * after semantic header/footer filtering. `textFragments` carries every
33
+ * survivor's native provenance and exact range in `text`; `stextJson` is the
34
+ * copied raw mupdf structured-text JSON for downstream visual detection.
35
+ */
36
+ interface ExtractedPage {
37
+ pageNum: number;
38
+ text: string;
39
+ textFragments: FilteredTextFragment[];
40
+ stextJson: StextJson;
41
+ }
42
+ /**
43
+ * Result returned by `extractPdfPages`. The helper lifts three concerns
44
+ * out of the legacy `parsePdf` body:
45
+ * 1. the per-page `toStructuredText` + `block.type === 'text'` loop;
46
+ * 2. `filterPageBoundarySentences` for header/footer removal;
47
+ * 3. title-resolution materials (`metadataTitle` and `page1FontHint`).
48
+ *
49
+ * Both `parsePdf` and `parsePdfPages` consume this helper; they differ only
50
+ * in the `stextOptions` argument they pass to `page.toStructuredText(...)`.
51
+ */
52
+ interface ExtractedPdf {
53
+ pages: ExtractedPage[];
54
+ metadataTitle: string | undefined;
55
+ page1FontHint: {
56
+ text: string;
57
+ fontSize: number;
58
+ } | undefined;
59
+ }
60
+ /**
61
+ * Per-page extraction shared by `parsePdf` and `parsePdfPages`.
62
+ *
63
+ * Takes an already-open mupdf `Document` and:
64
+ * - reads `info:Title` once,
65
+ * - iterates pages calling `toStructuredText(stextOptions)`,
66
+ * - builds `PageData` items (only `block.type === 'text'` lines),
67
+ * - runs `filterPageBoundarySentences` to drop semantic headers/footers,
68
+ * - derives `page1FontHint` from page 1's largest-font lines.
69
+ *
70
+ * The two callers differ ONLY in `stextOptions`: `parsePdf` passes
71
+ * `'preserve-whitespace'`;
72
+ * `parsePdfPages` passes `'preserve-whitespace,preserve-images'` so mupdf
73
+ * emits `block.type === 'image'` entries for the downstream visual-candidate
74
+ * detector.
75
+ *
76
+ * Lifecycle: this helper does NOT call `doc.destroy()` — disposal stays
77
+ * with the caller.
78
+ */
79
+ export declare function extractPdfPages(doc: MupdfDocument, embedder: EmbedderInterface, stextOptions: string): Promise<ExtractedPdf>;
80
+ export {};
81
+ //# sourceMappingURL=pdf-extract.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"pdf-extract.d.ts","sourceRoot":"","sources":["../../src/parser/pdf-extract.ts"],"names":[],"mappings":"AAUA,OAAO,KAAK,EAAE,QAAQ,IAAI,aAAa,EAAE,MAAM,OAAO,CAAA;AACtD,OAAO,EACL,KAAK,iBAAiB,EACtB,KAAK,oBAAoB,EAG1B,MAAM,iBAAiB,CAAA;AAExB,UAAU,SAAS;IACjB,CAAC,EAAE,MAAM,CAAA;IACT,CAAC,EAAE,MAAM,CAAA;IACT,CAAC,EAAE,MAAM,CAAA;IACT,CAAC,EAAE,MAAM,CAAA;CACV;AAED;;;;GAIG;AACH,UAAU,SAAS;IACjB,MAAM,EAAE,KAAK,CAAC;QACZ,IAAI,EAAE,MAAM,CAAA;QACZ,KAAK,CAAC,EAAE,KAAK,CAAC;YACZ,IAAI,EAAE,MAAM,CAAA;YACZ,CAAC,EAAE,MAAM,CAAA;YACT,CAAC,EAAE,MAAM,CAAA;YACT,IAAI,CAAC,EAAE,SAAS,CAAA;YAChB,IAAI,EAAE;gBAAE,IAAI,EAAE,MAAM,CAAC;gBAAC,IAAI,CAAC,EAAE,MAAM,CAAC;gBAAC,MAAM,CAAC,EAAE,MAAM,CAAA;aAAE,CAAA;SACvD,CAAC,CAAA;KACH,CAAC,CAAA;CACH;AAED;;;;;GAKG;AACH,UAAU,aAAa;IACrB,OAAO,EAAE,MAAM,CAAA;IACf,IAAI,EAAE,MAAM,CAAA;IACZ,aAAa,EAAE,oBAAoB,EAAE,CAAA;IACrC,SAAS,EAAE,SAAS,CAAA;CACrB;AAED;;;;;;;;;GASG;AACH,UAAU,YAAY;IACpB,KAAK,EAAE,aAAa,EAAE,CAAA;IACtB,aAAa,EAAE,MAAM,GAAG,SAAS,CAAA;IACjC,aAAa,EAAE;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAA;KAAE,GAAG,SAAS,CAAA;CAC9D;AAED;;;;;;;;;;;;;;;;;;GAkBG;AACH,wBAAsB,eAAe,CACnC,GAAG,EAAE,aAAa,EAClB,QAAQ,EAAE,iBAAiB,EAC3B,YAAY,EAAE,MAAM,GACnB,OAAO,CAAC,YAAY,CAAC,CAoFvB"}