agent-rag-mcp 0.0.0-stage → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,214 @@
1
+ import { PassThrough } from "node:stream";
2
+ import { StringDecoder } from "node:string_decoder";
3
+ import JSZip from "jszip";
4
+ import { ResourceLimitError } from "../ports/extractor.mjs";
5
+ const PPTX_MEDIA_TYPE = "application/vnd.openxmlformats-officedocument.presentationml.presentation";
6
+ const MAX_PPTX_ARCHIVE_ENTRIES = 5e3;
7
+ const MAX_PPTX_UNCOMPRESSED_BYTES = 128 * 1024 * 1024;
8
+ function supports(ref) {
9
+ if (typeof ref === "string") return /\.pptx$/i.test(ref);
10
+ if (typeof ref !== "object" || ref === null) return false;
11
+ const fields = ref;
12
+ const mediaType = fields.mediaType ?? fields.mimeType ?? fields.contentType;
13
+ if (typeof mediaType === "string") return mediaType.split(";")[0].trim().toLowerCase() === PPTX_MEDIA_TYPE;
14
+ const fileName = fields.filename ?? fields.name ?? fields.path ?? fields.extension;
15
+ return typeof fileName === "string" && /(?:^|[\\/])[^\\/]+\.pptx$/i.test(fileName);
16
+ }
17
+ function decodeXmlText(text) {
18
+ return text.replace(/&(#x[\da-f]+|#\d+|amp|lt|gt|quot|apos);/gi, (entity, reference) => {
19
+ switch (reference.toLowerCase()) {
20
+ case "amp":
21
+ return "&";
22
+ case "lt":
23
+ return "<";
24
+ case "gt":
25
+ return ">";
26
+ case "quot":
27
+ return '"';
28
+ case "apos":
29
+ return "'";
30
+ default: {
31
+ const codePoint = reference.startsWith("#x") || reference.startsWith("#X") ? Number.parseInt(reference.slice(2), 16) : Number.parseInt(reference.slice(1), 10);
32
+ return String.fromCodePoint(codePoint);
33
+ }
34
+ }
35
+ });
36
+ }
37
+ function slideNumber(fileName) {
38
+ return Number(fileName.match(/\/slide(\d+)\.xml$/)?.[1] ?? Number.MAX_SAFE_INTEGER);
39
+ }
40
+ function rejectLargeArchive(message) {
41
+ throw new ResourceLimitError("archive_too_large", message);
42
+ }
43
+ function assertPptxReadWithinLimits(fileName, entryBytesRead, declaredSize, totalBytesRead) {
44
+ if (entryBytesRead > declaredSize) {
45
+ rejectLargeArchive(`PPTX entry ${fileName} decompressed size differs from its declared size (${declaredSize} bytes).`);
46
+ }
47
+ if (totalBytesRead > MAX_PPTX_UNCOMPRESSED_BYTES) {
48
+ rejectLargeArchive(`PPTX exceeds MAX_PPTX_UNCOMPRESSED_BYTES (${MAX_PPTX_UNCOMPRESSED_BYTES} bytes).`);
49
+ }
50
+ }
51
+ function zip64Value(view, offset, end) {
52
+ if (offset + 8 > end) rejectLargeArchive("PPTX ZIP64 metadata is truncated.");
53
+ const value = view.getBigUint64(offset, true);
54
+ if (value > BigInt(Number.MAX_SAFE_INTEGER)) rejectLargeArchive("PPTX ZIP64 metadata exceeds safe bounds.");
55
+ return Number(value);
56
+ }
57
+ function hasSignature(view, offset, signature) {
58
+ return offset >= 0 && offset + 4 <= view.byteLength && view.getUint32(offset, true) === signature;
59
+ }
60
+ function zip64EndOffset(view, locatorOffset) {
61
+ const relativeOffset = zip64Value(view, locatorOffset + 8, locatorOffset + 20);
62
+ const validRecord = (offset) => {
63
+ if (!hasSignature(view, offset, 101075792) || offset + 12 > locatorOffset) return false;
64
+ const recordSize = zip64Value(view, offset + 4, locatorOffset);
65
+ return recordSize >= 44 && offset + 12 + recordSize === locatorOffset;
66
+ };
67
+ if (validRecord(relativeOffset)) return relativeOffset;
68
+ for (let offset = locatorOffset - 56; offset >= 0; offset -= 1) {
69
+ if (validRecord(offset)) return offset;
70
+ }
71
+ rejectLargeArchive("PPTX ZIP64 end record cannot be located safely.");
72
+ }
73
+ function centralUncompressedSize(view, offset, end, nameLength, extraLength) {
74
+ const size = view.getUint32(offset + 24, true);
75
+ if (size !== 4294967295) return size;
76
+ let extraOffset = offset + 46 + nameLength;
77
+ const extraEnd = extraOffset + extraLength;
78
+ while (extraOffset + 4 <= extraEnd) {
79
+ const identifier = view.getUint16(extraOffset, true);
80
+ const fieldLength = view.getUint16(extraOffset + 2, true);
81
+ const valueOffset = extraOffset + 4;
82
+ const fieldEnd = valueOffset + fieldLength;
83
+ if (fieldEnd > extraEnd || fieldEnd > end) rejectLargeArchive("PPTX central-directory extra field is truncated.");
84
+ if (identifier === 1) return zip64Value(view, valueOffset, fieldEnd);
85
+ extraOffset = fieldEnd;
86
+ }
87
+ rejectLargeArchive("PPTX ZIP64 entry size is missing from its central-directory extra field.");
88
+ }
89
+ function checkArchiveLimits(bytes) {
90
+ const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
91
+ const firstEndRecord = Math.max(0, bytes.byteLength - 22 - 65535);
92
+ let endRecord = -1;
93
+ for (let offset2 = bytes.byteLength - 22; offset2 >= firstEndRecord; offset2 -= 1) {
94
+ if (hasSignature(view, offset2, 101010256) && offset2 + 22 + view.getUint16(offset2 + 20, true) === bytes.byteLength) {
95
+ endRecord = offset2;
96
+ break;
97
+ }
98
+ }
99
+ if (endRecord < 0) return /* @__PURE__ */ new Map();
100
+ const classicCount = view.getUint16(endRecord + 10, true);
101
+ const classicSize = view.getUint32(endRecord + 12, true);
102
+ const classicOffset = view.getUint32(endRecord + 16, true);
103
+ const locatorOffset = endRecord - 20;
104
+ const hasZip64 = hasSignature(view, locatorOffset, 117853008);
105
+ const hasSentinel = classicCount === 65535 || classicSize === 4294967295 || classicOffset === 4294967295;
106
+ if (hasSentinel && !hasZip64) rejectLargeArchive("PPTX ZIP64 end record is missing.");
107
+ let entryCount = classicCount;
108
+ let centralSize = classicSize;
109
+ let centralOffset = classicOffset;
110
+ let directoryEnd = endRecord;
111
+ if (hasZip64) {
112
+ const zip64End = zip64EndOffset(view, locatorOffset);
113
+ const recordEnd = locatorOffset;
114
+ entryCount = zip64Value(view, zip64End + 32, recordEnd);
115
+ centralSize = zip64Value(view, zip64End + 40, recordEnd);
116
+ centralOffset = zip64Value(view, zip64End + 48, recordEnd);
117
+ directoryEnd = zip64End;
118
+ }
119
+ if (entryCount > MAX_PPTX_ARCHIVE_ENTRIES) {
120
+ rejectLargeArchive(`PPTX exceeds MAX_PPTX_ARCHIVE_ENTRIES (${MAX_PPTX_ARCHIVE_ENTRIES}).`);
121
+ }
122
+ if (centralSize > directoryEnd || centralOffset > directoryEnd || centralSize + centralOffset > directoryEnd) {
123
+ rejectLargeArchive("PPTX central-directory bounds cannot be checked safely.");
124
+ }
125
+ const prefixLength = directoryEnd - centralSize - centralOffset;
126
+ let offset = centralOffset + prefixLength;
127
+ const directoryLimit = offset + centralSize;
128
+ let actualEntryCount = 0;
129
+ let totalUncompressed = 0;
130
+ const declaredSizes = /* @__PURE__ */ new Map();
131
+ while (offset + 4 <= directoryLimit && hasSignature(view, offset, 33639248)) {
132
+ if (offset + 46 > directoryLimit) rejectLargeArchive("PPTX central-directory bounds cannot be checked safely.");
133
+ const nameLength = view.getUint16(offset + 28, true);
134
+ const extraLength = view.getUint16(offset + 30, true);
135
+ const commentLength = view.getUint16(offset + 32, true);
136
+ const nextOffset = offset + 46 + nameLength + extraLength + commentLength;
137
+ if (nextOffset > directoryLimit) rejectLargeArchive("PPTX central-directory entry is truncated.");
138
+ actualEntryCount += 1;
139
+ if (actualEntryCount > MAX_PPTX_ARCHIVE_ENTRIES) {
140
+ rejectLargeArchive(`PPTX exceeds MAX_PPTX_ARCHIVE_ENTRIES (${MAX_PPTX_ARCHIVE_ENTRIES}).`);
141
+ }
142
+ const uncompressedSize = centralUncompressedSize(view, offset, directoryLimit, nameLength, extraLength);
143
+ const name = new TextDecoder().decode(bytes.subarray(offset + 46, offset + 46 + nameLength));
144
+ declaredSizes.set(name, uncompressedSize);
145
+ if (uncompressedSize > MAX_PPTX_UNCOMPRESSED_BYTES - totalUncompressed) {
146
+ rejectLargeArchive(`PPTX exceeds MAX_PPTX_UNCOMPRESSED_BYTES (${MAX_PPTX_UNCOMPRESSED_BYTES} bytes).`);
147
+ }
148
+ totalUncompressed += uncompressedSize;
149
+ offset = nextOffset;
150
+ }
151
+ if (offset < directoryLimit && hasSignature(view, offset, 84233040) && offset + 6 <= directoryLimit) {
152
+ const signatureLength = view.getUint16(offset + 4, true);
153
+ if (offset + 6 + signatureLength === directoryLimit) offset = directoryLimit;
154
+ }
155
+ if (offset !== directoryLimit || actualEntryCount !== entryCount) {
156
+ rejectLargeArchive("PPTX central-directory count or bounds cannot be checked safely.");
157
+ }
158
+ return declaredSizes;
159
+ }
160
+ async function readSlideXml(file, alreadyRead, declaredSize) {
161
+ const chunks = [];
162
+ const decoder = new StringDecoder("utf8");
163
+ const stream = new PassThrough();
164
+ const source = file.nodeStream("nodebuffer");
165
+ source.on("error", (error) => stream.destroy(error));
166
+ source.pipe(stream);
167
+ let bytesRead = 0;
168
+ try {
169
+ for await (const rawChunk of stream) {
170
+ const chunk = Buffer.isBuffer(rawChunk) ? rawChunk : Buffer.from(rawChunk);
171
+ bytesRead += chunk.byteLength;
172
+ assertPptxReadWithinLimits(file.name, bytesRead, declaredSize, alreadyRead + bytesRead);
173
+ chunks.push(decoder.write(chunk));
174
+ }
175
+ } catch (error) {
176
+ if (error instanceof Error && error.message.includes("uncompressed data size mismatch")) {
177
+ rejectLargeArchive(`PPTX entry ${file.name} decompressed size differs from its declared size (${declaredSize} bytes).`);
178
+ }
179
+ throw error;
180
+ }
181
+ chunks.push(decoder.end());
182
+ return { xml: chunks.join(""), bytesRead };
183
+ }
184
+ async function extract(bytes) {
185
+ const declaredSizes = checkArchiveLimits(bytes);
186
+ const archive = await JSZip.loadAsync(bytes);
187
+ const slides = archive.file(/^ppt\/slides\/slide\d+\.xml$/).sort((left, right) => slideNumber(left.name) - slideNumber(right.name));
188
+ const sections = [];
189
+ let slideBytesRead = 0;
190
+ for (const slide of slides) {
191
+ const declaredSize = declaredSizes.get(slide.name);
192
+ if (declaredSize === void 0) {
193
+ rejectLargeArchive(`PPTX central-directory size for ${slide.name} cannot be checked safely.`);
194
+ }
195
+ const slideXml = await readSlideXml(slide, slideBytesRead, declaredSize);
196
+ slideBytesRead += slideXml.bytesRead;
197
+ const text = [...slideXml.xml.matchAll(/<a:t(?:\s[^>]*)?>([\s\S]*?)<\/a:t>/g)].map((match) => decodeXmlText(match[1] ?? "").trim()).filter((value) => value.length > 0).join(" ");
198
+ if (text.length > 0) sections.push({ ordinal: sections.length, heading: null, level: null, text });
199
+ }
200
+ return { sections, versionLabel: null, mediaType: PPTX_MEDIA_TYPE };
201
+ }
202
+ const pptxExtractor = {
203
+ name: "pptx",
204
+ version: "3.10.2",
205
+ mediaTypes: [PPTX_MEDIA_TYPE],
206
+ supports,
207
+ extract
208
+ };
209
+ export {
210
+ MAX_PPTX_ARCHIVE_ENTRIES,
211
+ MAX_PPTX_UNCOMPRESSED_BYTES,
212
+ assertPptxReadWithinLimits,
213
+ pptxExtractor
214
+ };
@@ -0,0 +1,45 @@
1
+ import * as path from "node:path";
2
+ import { docxExtractor } from "./docx.mjs";
3
+ import { imageOcrExtractor } from "./image-ocr.mjs";
4
+ import { pdfExtractor } from "./pdf.mjs";
5
+ import { plainTextExtractor } from "./plain-text.mjs";
6
+ import { jsonExtractor, jsonlExtractor, yamlExtractor } from "./structured-text.mjs";
7
+ import { pptxExtractor } from "./pptx.mjs";
8
+ import { xlsxExtractor } from "./xlsx.mjs";
9
+ class UnsupportedMediaTypeError extends Error {
10
+ code = "unsupported_media_type";
11
+ extension;
12
+ constructor(filePath) {
13
+ const extension = path.extname(filePath).toLowerCase() || "(none)";
14
+ super(`Unsupported media type for extension "${extension}".`);
15
+ this.name = "UnsupportedMediaTypeError";
16
+ this.extension = extension;
17
+ }
18
+ }
19
+ const extractorsByExtension = /* @__PURE__ */ new Map([
20
+ [".pdf", pdfExtractor],
21
+ [".docx", docxExtractor],
22
+ [".png", imageOcrExtractor],
23
+ [".jpg", imageOcrExtractor],
24
+ [".jpeg", imageOcrExtractor],
25
+ [".xlsx", xlsxExtractor],
26
+ [".pptx", pptxExtractor],
27
+ [".txt", plainTextExtractor],
28
+ [".md", plainTextExtractor],
29
+ [".yaml", yamlExtractor],
30
+ [".yml", yamlExtractor],
31
+ [".json", jsonExtractor],
32
+ [".jsonl", jsonlExtractor]
33
+ ]);
34
+ function getExtractorForPath(filePath) {
35
+ const extension = path.extname(filePath).toLowerCase();
36
+ const extractor = extractorsByExtension.get(extension);
37
+ if (!extractor || !extractor.supports({ filename: filePath })) {
38
+ throw new UnsupportedMediaTypeError(filePath);
39
+ }
40
+ return extractor;
41
+ }
42
+ export {
43
+ UnsupportedMediaTypeError,
44
+ getExtractorForPath
45
+ };
@@ -0,0 +1,150 @@
1
+ class StructuredTextExtractionError extends Error {
2
+ code;
3
+ constructor(code) {
4
+ super(code === "extraction_invalid_json" ? "Structured JSON is invalid." : "Structured YAML is invalid.");
5
+ this.name = "StructuredTextExtractionError";
6
+ this.code = code;
7
+ }
8
+ }
9
+ function getReferenceField(ref, names) {
10
+ if (typeof ref === "string") return ref;
11
+ if (typeof ref !== "object" || ref === null) return null;
12
+ const value = ref;
13
+ for (const name of names) {
14
+ if (typeof value[name] === "string") return value[name];
15
+ }
16
+ return null;
17
+ }
18
+ function supportsReference(ref, options) {
19
+ if (typeof ref !== "string") {
20
+ const mediaType = getReferenceField(ref, ["mediaType", "mimeType", "contentType"]);
21
+ if (mediaType !== null) {
22
+ return options.mediaTypes.includes(mediaType.split(";")[0].trim().toLowerCase());
23
+ }
24
+ }
25
+ const fileName = typeof ref === "string" ? ref : getReferenceField(ref, ["filename", "name", "path", "extension"]);
26
+ if (fileName === null) return false;
27
+ const normalizedFileName = fileName.toLowerCase();
28
+ return options.extensions.some((extension) => normalizedFileName.endsWith(extension));
29
+ }
30
+ function renderJsonValue(value) {
31
+ if (typeof value === "string") return value;
32
+ return JSON.stringify(value, null, 2) ?? "null";
33
+ }
34
+ function extractJson(content) {
35
+ let parsed;
36
+ try {
37
+ parsed = JSON.parse(content);
38
+ } catch {
39
+ throw new StructuredTextExtractionError("extraction_invalid_json");
40
+ }
41
+ if (typeof parsed === "object" && parsed !== null && !Array.isArray(parsed)) {
42
+ return Object.entries(parsed).map(([heading, value], ordinal) => ({
43
+ ordinal,
44
+ heading,
45
+ level: 1,
46
+ text: renderJsonValue(value)
47
+ }));
48
+ }
49
+ return [{ ordinal: 0, heading: null, level: null, text: renderJsonValue(parsed) }];
50
+ }
51
+ function extractJsonl(content) {
52
+ const sections = [];
53
+ for (const line of content.split(/\r?\n/)) {
54
+ const text = line.trim();
55
+ if (text.length === 0) continue;
56
+ try {
57
+ JSON.parse(text);
58
+ } catch {
59
+ throw new StructuredTextExtractionError("extraction_invalid_json");
60
+ }
61
+ sections.push({ ordinal: sections.length, heading: null, level: null, text });
62
+ }
63
+ return sections;
64
+ }
65
+ function extractYaml(content) {
66
+ const lines = content.split("\n").map((line) => line.replace(/\r$/, ""));
67
+ const topLevelKey = /^([A-Za-z0-9_.-]+):/;
68
+ const starts = [];
69
+ for (let index = 0; index < lines.length; index += 1) {
70
+ const line = lines[index];
71
+ if (line === void 0) continue;
72
+ if (/^[ \t]*\t/.test(line)) throw new StructuredTextExtractionError("extraction_invalid_yaml");
73
+ const match = topLevelKey.exec(line);
74
+ if (match?.[1] !== void 0) starts.push({ index, heading: match[1] });
75
+ }
76
+ if (starts.length === 0) {
77
+ return [{ ordinal: 0, heading: null, level: null, text: content }];
78
+ }
79
+ let hasCurrentBlock = false;
80
+ for (const line of lines) {
81
+ const trimmed = line.trim();
82
+ if (trimmed.length === 0 || trimmed.startsWith("#") || trimmed === "---" || trimmed === "...") continue;
83
+ const match = topLevelKey.exec(line);
84
+ if (match?.[1] !== void 0) {
85
+ hasCurrentBlock = true;
86
+ continue;
87
+ }
88
+ const indentation = line.match(/^ */)?.[0].length ?? 0;
89
+ if (indentation === 0 || !hasCurrentBlock) throw new StructuredTextExtractionError("extraction_invalid_yaml");
90
+ }
91
+ return starts.map((start, ordinal) => {
92
+ const nextStart = starts[ordinal + 1];
93
+ const block = lines.slice(start.index, nextStart?.index ?? lines.length).join("\n").trim();
94
+ return { ordinal, heading: start.heading, level: 1, text: block };
95
+ });
96
+ }
97
+ function createStructuredTextExtractor(options) {
98
+ return {
99
+ name: options.name,
100
+ version: "1.0.0",
101
+ mediaTypes: options.mediaTypes,
102
+ supports(ref) {
103
+ return supportsReference(ref, options);
104
+ },
105
+ async extract(bytes) {
106
+ const decoded = new TextDecoder("utf-8", { fatal: false, ignoreBOM: true }).decode(bytes);
107
+ const content = decoded.startsWith("\uFEFF") ? decoded.slice(1) : decoded;
108
+ if (content.length === 0) {
109
+ return { sections: [], versionLabel: null, mediaType: null };
110
+ }
111
+ let sections;
112
+ switch (options.kind) {
113
+ case "json":
114
+ sections = extractJson(content);
115
+ break;
116
+ case "jsonl":
117
+ sections = extractJsonl(content);
118
+ break;
119
+ case "yaml":
120
+ sections = extractYaml(content);
121
+ break;
122
+ }
123
+ return { sections, versionLabel: null, mediaType: options.mediaTypes[0] ?? null };
124
+ }
125
+ };
126
+ }
127
+ const jsonExtractor = createStructuredTextExtractor({
128
+ kind: "json",
129
+ name: "json",
130
+ extensions: [".json"],
131
+ mediaTypes: ["application/json"]
132
+ });
133
+ const jsonlExtractor = createStructuredTextExtractor({
134
+ kind: "jsonl",
135
+ name: "jsonl",
136
+ extensions: [".jsonl"],
137
+ mediaTypes: ["application/x-ndjson"]
138
+ });
139
+ const yamlExtractor = createStructuredTextExtractor({
140
+ kind: "yaml",
141
+ name: "yaml",
142
+ extensions: [".yaml", ".yml"],
143
+ mediaTypes: ["application/yaml"]
144
+ });
145
+ export {
146
+ StructuredTextExtractionError,
147
+ jsonExtractor,
148
+ jsonlExtractor,
149
+ yamlExtractor
150
+ };