@qwen-code/arena-agent 0.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.env.example +52 -0
- package/README.md +436 -0
- package/dist/agent.d.ts +33 -0
- package/dist/agent.js +366 -0
- package/dist/agent.js.map +1 -0
- package/dist/attachments.d.ts +29 -0
- package/dist/attachments.js +222 -0
- package/dist/attachments.js.map +1 -0
- package/dist/cli.d.ts +2 -0
- package/dist/cli.js +256 -0
- package/dist/cli.js.map +1 -0
- package/dist/compaction.d.ts +15 -0
- package/dist/compaction.js +176 -0
- package/dist/compaction.js.map +1 -0
- package/dist/config.d.ts +23 -0
- package/dist/config.js +61 -0
- package/dist/config.js.map +1 -0
- package/dist/document-reader.d.ts +18 -0
- package/dist/document-reader.js +423 -0
- package/dist/document-reader.js.map +1 -0
- package/dist/fetch-page.d.ts +25 -0
- package/dist/fetch-page.js +428 -0
- package/dist/fetch-page.js.map +1 -0
- package/dist/history.d.ts +5 -0
- package/dist/history.js +31 -0
- package/dist/history.js.map +1 -0
- package/dist/mock.d.ts +4 -0
- package/dist/mock.js +151 -0
- package/dist/mock.js.map +1 -0
- package/dist/protocol.d.ts +23 -0
- package/dist/protocol.js +84 -0
- package/dist/protocol.js.map +1 -0
- package/dist/provider.d.ts +31 -0
- package/dist/provider.js +308 -0
- package/dist/provider.js.map +1 -0
- package/dist/sdk.d.ts +181 -0
- package/dist/sdk.js +674 -0
- package/dist/sdk.js.map +1 -0
- package/dist/server.d.ts +15 -0
- package/dist/server.js +1433 -0
- package/dist/server.js.map +1 -0
- package/dist/spec.d.ts +11 -0
- package/dist/spec.js +198 -0
- package/dist/spec.js.map +1 -0
- package/dist/tools.d.ts +38 -0
- package/dist/tools.js +992 -0
- package/dist/tools.js.map +1 -0
- package/dist/tui.d.ts +10 -0
- package/dist/tui.js +350 -0
- package/dist/tui.js.map +1 -0
- package/dist/types.d.ts +146 -0
- package/dist/types.js +2 -0
- package/dist/types.js.map +1 -0
- package/dist/web/app.d.ts +1 -0
- package/dist/web/app.js +1082 -0
- package/dist/web/app.js.map +1 -0
- package/docs/SDK.md +521 -0
- package/package.json +59 -0
- package/spec/message.json +150 -0
- package/spec/tools.json +418 -0
- package/web/arena-logo.svg +11 -0
- package/web/fonts.css +67 -0
- package/web/index.html +285 -0
- package/web/styles.css +1798 -0
|
@@ -0,0 +1,423 @@
|
|
|
1
|
+
import { spawn } from "node:child_process";
|
|
2
|
+
import { mkdir, mkdtemp, readFile, readdir, rm } from "node:fs/promises";
|
|
3
|
+
import { tmpdir } from "node:os";
|
|
4
|
+
import path from "node:path";
|
|
5
|
+
import { pathToFileURL } from "node:url";
|
|
6
|
+
import { strFromU8, unzipSync } from "fflate";
|
|
7
|
+
import { PDFParse } from "pdf-parse";
|
|
8
|
+
const MAX_ARCHIVE_ENTRY_BYTES = 32 * 1024 * 1024;
|
|
9
|
+
const MAX_ARCHIVE_TOTAL_BYTES = 64 * 1024 * 1024;
|
|
10
|
+
const MAX_EXTRACTED_TEXT_CHARS = 500_000;
|
|
11
|
+
const MAX_SPREADSHEET_CELLS = 100_000;
|
|
12
|
+
const OFFICE_CONVERSION_TIMEOUT_MS = 30_000;
|
|
13
|
+
const structuredDocumentExtensions = new Set([
|
|
14
|
+
".doc",
|
|
15
|
+
".docx",
|
|
16
|
+
".pdf",
|
|
17
|
+
".ppt",
|
|
18
|
+
".pptx",
|
|
19
|
+
".xls",
|
|
20
|
+
".xlsx",
|
|
21
|
+
]);
|
|
22
|
+
const legacyOfficeTargets = {
|
|
23
|
+
".doc": ".docx",
|
|
24
|
+
".ppt": ".pptx",
|
|
25
|
+
".xls": ".xlsx",
|
|
26
|
+
};
|
|
27
|
+
function normalizeExtractedText(value) {
|
|
28
|
+
return value
|
|
29
|
+
.replace(/\r\n?/g, "\n")
|
|
30
|
+
.replace(/[^\S\n]+\n/g, "\n")
|
|
31
|
+
.replace(/\n{3,}/g, "\n\n")
|
|
32
|
+
.trim();
|
|
33
|
+
}
|
|
34
|
+
function limitedContent(value) {
|
|
35
|
+
const content = normalizeExtractedText(value);
|
|
36
|
+
if (content.length <= MAX_EXTRACTED_TEXT_CHARS)
|
|
37
|
+
return { content };
|
|
38
|
+
return {
|
|
39
|
+
content: content.slice(0, MAX_EXTRACTED_TEXT_CHARS) +
|
|
40
|
+
"\n\n[Extracted document text truncated.]",
|
|
41
|
+
truncated: true,
|
|
42
|
+
};
|
|
43
|
+
}
|
|
44
|
+
function decodeXmlEntities(value) {
|
|
45
|
+
return value.replace(/&(?:#(\d+)|#x([0-9a-f]+)|amp|apos|gt|lt|quot);/gi, (entity, decimal, hexadecimal) => {
|
|
46
|
+
if (decimal)
|
|
47
|
+
return String.fromCodePoint(Number(decimal));
|
|
48
|
+
if (hexadecimal)
|
|
49
|
+
return String.fromCodePoint(Number.parseInt(hexadecimal, 16));
|
|
50
|
+
switch (entity.toLowerCase()) {
|
|
51
|
+
case "&":
|
|
52
|
+
return "&";
|
|
53
|
+
case "'":
|
|
54
|
+
return "'";
|
|
55
|
+
case ">":
|
|
56
|
+
return ">";
|
|
57
|
+
case "<":
|
|
58
|
+
return "<";
|
|
59
|
+
case """:
|
|
60
|
+
return '"';
|
|
61
|
+
default:
|
|
62
|
+
return entity;
|
|
63
|
+
}
|
|
64
|
+
});
|
|
65
|
+
}
|
|
66
|
+
function xmlAttribute(tag, name) {
|
|
67
|
+
const escaped = name.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
|
68
|
+
const match = tag.match(new RegExp(`(?:^|\\s)${escaped}\\s*=\\s*(?:"([^"]*)"|'([^']*)')`, "i"));
|
|
69
|
+
return decodeXmlEntities(match?.[1] || match?.[2] || "");
|
|
70
|
+
}
|
|
71
|
+
function xmlText(value) {
|
|
72
|
+
return decodeXmlEntities(value.replace(/<[^>]+>/g, ""));
|
|
73
|
+
}
|
|
74
|
+
function archiveEntries(data, include) {
|
|
75
|
+
let expandedBytes = 0;
|
|
76
|
+
try {
|
|
77
|
+
return unzipSync(data, {
|
|
78
|
+
filter(file) {
|
|
79
|
+
const name = file.name.replaceAll("\\", "/");
|
|
80
|
+
if (!include(name))
|
|
81
|
+
return false;
|
|
82
|
+
if (file.originalSize > MAX_ARCHIVE_ENTRY_BYTES) {
|
|
83
|
+
throw new Error(`archive entry is too large: ${name}`);
|
|
84
|
+
}
|
|
85
|
+
expandedBytes += file.originalSize;
|
|
86
|
+
if (expandedBytes > MAX_ARCHIVE_TOTAL_BYTES) {
|
|
87
|
+
throw new Error("document expands beyond the safe extraction limit");
|
|
88
|
+
}
|
|
89
|
+
return true;
|
|
90
|
+
},
|
|
91
|
+
});
|
|
92
|
+
}
|
|
93
|
+
catch (error) {
|
|
94
|
+
throw new Error(`could not unpack Office document: ${errorMessage(error)}`);
|
|
95
|
+
}
|
|
96
|
+
}
|
|
97
|
+
function entryText(entries, name) {
|
|
98
|
+
const bytes = entries[name];
|
|
99
|
+
return bytes ? strFromU8(bytes) : "";
|
|
100
|
+
}
|
|
101
|
+
function numberedPathSort(left, right) {
|
|
102
|
+
const leftNumber = Number(left.match(/(\d+)(?=\.xml$)/)?.[1] || 0);
|
|
103
|
+
const rightNumber = Number(right.match(/(\d+)(?=\.xml$)/)?.[1] || 0);
|
|
104
|
+
return leftNumber - rightNumber || left.localeCompare(right);
|
|
105
|
+
}
|
|
106
|
+
function extractDocx(data) {
|
|
107
|
+
const entries = archiveEntries(data, (name) => name === "word/document.xml" ||
|
|
108
|
+
/^word\/(?:header|footer)\d+\.xml$/.test(name) ||
|
|
109
|
+
/^word\/(?:footnotes|endnotes|comments)\.xml$/.test(name));
|
|
110
|
+
if (!entries["word/document.xml"]) {
|
|
111
|
+
throw new Error("DOCX does not contain word/document.xml");
|
|
112
|
+
}
|
|
113
|
+
const extractPart = (xml) => normalizeExtractedText(decodeXmlEntities(xml
|
|
114
|
+
.replace(/<w:tab\b[^>]*\/>/gi, "\t")
|
|
115
|
+
.replace(/<w:br\b[^>]*\/>/gi, "\n")
|
|
116
|
+
.replace(/<\/w:tc>/gi, "\t")
|
|
117
|
+
.replace(/<\/w:(?:p|tr)>/gi, "\n")
|
|
118
|
+
.replace(/<[^>]+>/g, "")));
|
|
119
|
+
const sections = [];
|
|
120
|
+
const main = extractPart(entryText(entries, "word/document.xml"));
|
|
121
|
+
if (main)
|
|
122
|
+
sections.push(main);
|
|
123
|
+
for (const name of Object.keys(entries)
|
|
124
|
+
.filter((candidate) => candidate !== "word/document.xml")
|
|
125
|
+
.sort(numberedPathSort)) {
|
|
126
|
+
const content = extractPart(entryText(entries, name));
|
|
127
|
+
if (!content)
|
|
128
|
+
continue;
|
|
129
|
+
const label = path.basename(name, ".xml").replace(/(\D)(\d+)/, "$1 $2");
|
|
130
|
+
sections.push(`## ${label}\n\n${content}`);
|
|
131
|
+
}
|
|
132
|
+
return sections.join("\n\n") || "_No extractable text._";
|
|
133
|
+
}
|
|
134
|
+
function extractPptx(data) {
|
|
135
|
+
const entries = archiveEntries(data, (name) => /^ppt\/slides\/slide\d+\.xml$/.test(name) ||
|
|
136
|
+
/^ppt\/notesSlides\/notesSlide\d+\.xml$/.test(name));
|
|
137
|
+
const slides = Object.keys(entries)
|
|
138
|
+
.filter((name) => /^ppt\/slides\/slide\d+\.xml$/.test(name))
|
|
139
|
+
.sort(numberedPathSort);
|
|
140
|
+
if (!slides.length)
|
|
141
|
+
throw new Error("PPTX does not contain any slides");
|
|
142
|
+
const extractPart = (xml) => normalizeExtractedText(decodeXmlEntities(xml
|
|
143
|
+
.replace(/<a:tab\b[^>]*\/>/gi, "\t")
|
|
144
|
+
.replace(/<a:br\b[^>]*\/>/gi, "\n")
|
|
145
|
+
.replace(/<\/a:p>/gi, "\n")
|
|
146
|
+
.replace(/<[^>]+>/g, "")));
|
|
147
|
+
return slides
|
|
148
|
+
.map((name) => {
|
|
149
|
+
const number = Number(name.match(/slide(\d+)\.xml$/)?.[1] || 0);
|
|
150
|
+
const content = extractPart(entryText(entries, name));
|
|
151
|
+
const notesName = `ppt/notesSlides/notesSlide${number}.xml`;
|
|
152
|
+
const notes = extractPart(entryText(entries, notesName));
|
|
153
|
+
return [
|
|
154
|
+
`## Slide ${number}`,
|
|
155
|
+
content || "_No extractable text._",
|
|
156
|
+
...(notes ? [`### Speaker notes\n\n${notes}`] : []),
|
|
157
|
+
].join("\n\n");
|
|
158
|
+
})
|
|
159
|
+
.join("\n\n");
|
|
160
|
+
}
|
|
161
|
+
function spreadsheetColumn(reference) {
|
|
162
|
+
const letters = reference.match(/^[A-Z]+/i)?.[0]?.toUpperCase() || "";
|
|
163
|
+
let column = 0;
|
|
164
|
+
for (const letter of letters) {
|
|
165
|
+
column = column * 26 + letter.charCodeAt(0) - 64;
|
|
166
|
+
}
|
|
167
|
+
return Math.max(0, column - 1);
|
|
168
|
+
}
|
|
169
|
+
function relationshipTarget(target) {
|
|
170
|
+
const normalized = target.replaceAll("\\", "/").replace(/^\/+/, "");
|
|
171
|
+
if (normalized.startsWith("xl/"))
|
|
172
|
+
return normalized;
|
|
173
|
+
const parts = [];
|
|
174
|
+
for (const part of `xl/${normalized}`.split("/")) {
|
|
175
|
+
if (part === "..")
|
|
176
|
+
parts.pop();
|
|
177
|
+
else if (part && part !== ".")
|
|
178
|
+
parts.push(part);
|
|
179
|
+
}
|
|
180
|
+
return parts.join("/");
|
|
181
|
+
}
|
|
182
|
+
function extractXlsx(data) {
|
|
183
|
+
const entries = archiveEntries(data, (name) => name === "xl/workbook.xml" ||
|
|
184
|
+
name === "xl/_rels/workbook.xml.rels" ||
|
|
185
|
+
name === "xl/sharedStrings.xml" ||
|
|
186
|
+
/^xl\/worksheets\/sheet\d+\.xml$/.test(name));
|
|
187
|
+
const worksheetPaths = Object.keys(entries)
|
|
188
|
+
.filter((name) => /^xl\/worksheets\/sheet\d+\.xml$/.test(name))
|
|
189
|
+
.sort(numberedPathSort);
|
|
190
|
+
if (!worksheetPaths.length) {
|
|
191
|
+
throw new Error("XLSX does not contain any worksheets");
|
|
192
|
+
}
|
|
193
|
+
const sharedStrings = [
|
|
194
|
+
...entryText(entries, "xl/sharedStrings.xml").matchAll(/<si\b[^>]*>([\s\S]*?)<\/si>/gi),
|
|
195
|
+
].map((match) => xmlText(match[1] || ""));
|
|
196
|
+
const relationships = new Map();
|
|
197
|
+
for (const match of entryText(entries, "xl/_rels/workbook.xml.rels").matchAll(/<Relationship\b([^>]*)\/?>/gi)) {
|
|
198
|
+
const id = xmlAttribute(match[1] || "", "Id");
|
|
199
|
+
const target = xmlAttribute(match[1] || "", "Target");
|
|
200
|
+
if (id && target)
|
|
201
|
+
relationships.set(id, relationshipTarget(target));
|
|
202
|
+
}
|
|
203
|
+
const sheetNames = new Map();
|
|
204
|
+
for (const match of entryText(entries, "xl/workbook.xml").matchAll(/<sheet\b([^>]*)\/?>/gi)) {
|
|
205
|
+
const attributes = match[1] || "";
|
|
206
|
+
const id = xmlAttribute(attributes, "r:id");
|
|
207
|
+
const name = xmlAttribute(attributes, "name");
|
|
208
|
+
const target = relationships.get(id);
|
|
209
|
+
if (target && name)
|
|
210
|
+
sheetNames.set(target, name);
|
|
211
|
+
}
|
|
212
|
+
let cellsSeen = 0;
|
|
213
|
+
return worksheetPaths
|
|
214
|
+
.map((worksheetPath, sheetIndex) => {
|
|
215
|
+
const rows = new Map();
|
|
216
|
+
const xml = entryText(entries, worksheetPath);
|
|
217
|
+
for (const match of xml.matchAll(/<c\b([^>]*)>([\s\S]*?)<\/c>/gi)) {
|
|
218
|
+
cellsSeen += 1;
|
|
219
|
+
if (cellsSeen > MAX_SPREADSHEET_CELLS) {
|
|
220
|
+
throw new Error(`spreadsheet exceeds ${MAX_SPREADSHEET_CELLS} readable cells`);
|
|
221
|
+
}
|
|
222
|
+
const attributes = match[1] || "";
|
|
223
|
+
const body = match[2] || "";
|
|
224
|
+
const reference = xmlAttribute(attributes, "r");
|
|
225
|
+
const type = xmlAttribute(attributes, "t");
|
|
226
|
+
const rowNumber = Number(reference.match(/\d+$/)?.[0] || 1);
|
|
227
|
+
const column = spreadsheetColumn(reference);
|
|
228
|
+
const rawValue = decodeXmlEntities(body.match(/<v\b[^>]*>([\s\S]*?)<\/v>/i)?.[1] || "");
|
|
229
|
+
const formula = decodeXmlEntities(body.match(/<f\b[^>]*>([\s\S]*?)<\/f>/i)?.[1] || "");
|
|
230
|
+
let value;
|
|
231
|
+
if (type === "s") {
|
|
232
|
+
value = sharedStrings[Number(rawValue)] || "";
|
|
233
|
+
}
|
|
234
|
+
else if (type === "inlineStr") {
|
|
235
|
+
value = xmlText(body.match(/<is\b[^>]*>([\s\S]*?)<\/is>/i)?.[1] || "");
|
|
236
|
+
}
|
|
237
|
+
else if (type === "b") {
|
|
238
|
+
value = rawValue === "1" ? "TRUE" : "FALSE";
|
|
239
|
+
}
|
|
240
|
+
else {
|
|
241
|
+
value = rawValue;
|
|
242
|
+
}
|
|
243
|
+
if (formula) {
|
|
244
|
+
value = value ? `=${formula} => ${value}` : `=${formula}`;
|
|
245
|
+
}
|
|
246
|
+
const row = rows.get(rowNumber) || new Map();
|
|
247
|
+
row.set(column, value);
|
|
248
|
+
rows.set(rowNumber, row);
|
|
249
|
+
}
|
|
250
|
+
const lines = [...rows.entries()]
|
|
251
|
+
.sort(([left], [right]) => left - right)
|
|
252
|
+
.map(([, row]) => {
|
|
253
|
+
const maximumColumn = Math.max(...row.keys(), 0);
|
|
254
|
+
return Array.from({ length: maximumColumn + 1 }, (_, column) => row.get(column) || "").join("\t");
|
|
255
|
+
});
|
|
256
|
+
const name = sheetNames.get(worksheetPath) || `Sheet ${sheetIndex + 1}`;
|
|
257
|
+
return `## Sheet: ${name}\n\n${lines.join("\n") || "_No populated cells._"}`;
|
|
258
|
+
})
|
|
259
|
+
.join("\n\n");
|
|
260
|
+
}
|
|
261
|
+
function errorMessage(error) {
|
|
262
|
+
return error instanceof Error ? error.message : String(error);
|
|
263
|
+
}
|
|
264
|
+
async function runOfficeConversion(sourcePath, targetExtension, signal) {
|
|
265
|
+
const temporaryDirectory = await mkdtemp(path.join(tmpdir(), "arena-read-file-"));
|
|
266
|
+
const outputDirectory = path.join(temporaryDirectory, "output");
|
|
267
|
+
const profileDirectory = path.join(temporaryDirectory, "profile");
|
|
268
|
+
await Promise.all([
|
|
269
|
+
mkdir(outputDirectory, { recursive: true }),
|
|
270
|
+
mkdir(profileDirectory, { recursive: true }),
|
|
271
|
+
]);
|
|
272
|
+
const command = process.env.ARENA_SOFFICE_PATH || "soffice";
|
|
273
|
+
const arguments_ = [
|
|
274
|
+
"--headless",
|
|
275
|
+
"--nologo",
|
|
276
|
+
"--nolockcheck",
|
|
277
|
+
"--nodefault",
|
|
278
|
+
"--nofirststartwizard",
|
|
279
|
+
`-env:UserInstallation=${pathToFileURL(profileDirectory).href}`,
|
|
280
|
+
"--convert-to",
|
|
281
|
+
targetExtension.slice(1),
|
|
282
|
+
"--outdir",
|
|
283
|
+
outputDirectory,
|
|
284
|
+
sourcePath,
|
|
285
|
+
];
|
|
286
|
+
try {
|
|
287
|
+
await new Promise((resolve, reject) => {
|
|
288
|
+
const child = spawn(command, arguments_, {
|
|
289
|
+
stdio: ["ignore", "pipe", "pipe"],
|
|
290
|
+
});
|
|
291
|
+
let stderr = "";
|
|
292
|
+
let timedOut = false;
|
|
293
|
+
const timeout = setTimeout(() => {
|
|
294
|
+
timedOut = true;
|
|
295
|
+
child.kill("SIGKILL");
|
|
296
|
+
}, OFFICE_CONVERSION_TIMEOUT_MS);
|
|
297
|
+
const abort = () => child.kill("SIGTERM");
|
|
298
|
+
signal?.addEventListener("abort", abort, { once: true });
|
|
299
|
+
child.stderr.on("data", (chunk) => {
|
|
300
|
+
if (stderr.length < 64_000)
|
|
301
|
+
stderr += chunk.toString("utf8");
|
|
302
|
+
});
|
|
303
|
+
child.once("error", (error) => {
|
|
304
|
+
clearTimeout(timeout);
|
|
305
|
+
signal?.removeEventListener("abort", abort);
|
|
306
|
+
if (error.code === "ENOENT") {
|
|
307
|
+
reject(new Error("legacy Office files require LibreOffice (`soffice`); " +
|
|
308
|
+
`install it or set ARENA_SOFFICE_PATH`));
|
|
309
|
+
}
|
|
310
|
+
else {
|
|
311
|
+
reject(error);
|
|
312
|
+
}
|
|
313
|
+
});
|
|
314
|
+
child.once("close", (code) => {
|
|
315
|
+
clearTimeout(timeout);
|
|
316
|
+
signal?.removeEventListener("abort", abort);
|
|
317
|
+
if (signal?.aborted) {
|
|
318
|
+
reject(signal.reason || new Error("operation cancelled"));
|
|
319
|
+
}
|
|
320
|
+
else if (timedOut) {
|
|
321
|
+
reject(new Error("LibreOffice conversion timed out"));
|
|
322
|
+
}
|
|
323
|
+
else if (code !== 0) {
|
|
324
|
+
reject(new Error(`LibreOffice conversion failed (${code}): ${stderr.trim()}`));
|
|
325
|
+
}
|
|
326
|
+
else {
|
|
327
|
+
resolve();
|
|
328
|
+
}
|
|
329
|
+
});
|
|
330
|
+
});
|
|
331
|
+
const files = await readdir(outputDirectory);
|
|
332
|
+
const convertedName = files.find((name) => path.extname(name).toLowerCase() === targetExtension);
|
|
333
|
+
if (!convertedName) {
|
|
334
|
+
throw new Error("LibreOffice did not produce a converted document");
|
|
335
|
+
}
|
|
336
|
+
return {
|
|
337
|
+
path: path.join(outputDirectory, convertedName),
|
|
338
|
+
cleanup: () => rm(temporaryDirectory, { recursive: true, force: true }),
|
|
339
|
+
};
|
|
340
|
+
}
|
|
341
|
+
catch (error) {
|
|
342
|
+
await rm(temporaryDirectory, { recursive: true, force: true });
|
|
343
|
+
throw error;
|
|
344
|
+
}
|
|
345
|
+
}
|
|
346
|
+
export function isStructuredDocument(filePath) {
|
|
347
|
+
return structuredDocumentExtensions.has(path.extname(filePath).toLowerCase());
|
|
348
|
+
}
|
|
349
|
+
export async function extractPdfDocument(data, firstPages = 30) {
|
|
350
|
+
const parser = new PDFParse({ data: new Uint8Array(data) });
|
|
351
|
+
try {
|
|
352
|
+
const result = await parser.getText({
|
|
353
|
+
first: firstPages,
|
|
354
|
+
parseHyperlinks: true,
|
|
355
|
+
pageJoiner: "",
|
|
356
|
+
});
|
|
357
|
+
const content = normalizeExtractedText(result.pages
|
|
358
|
+
.map((page) => {
|
|
359
|
+
const text = normalizeExtractedText(page.text);
|
|
360
|
+
return `## Page ${page.num}\n\n${text || "_No extractable text._"}`;
|
|
361
|
+
})
|
|
362
|
+
.join("\n\n"));
|
|
363
|
+
const pagesParsed = result.pages.length;
|
|
364
|
+
const hasExtractableText = result.pages.some((page) => page.text.trim());
|
|
365
|
+
return {
|
|
366
|
+
content,
|
|
367
|
+
totalPages: result.total,
|
|
368
|
+
pagesParsed,
|
|
369
|
+
truncated: result.total > pagesParsed,
|
|
370
|
+
...(!hasExtractableText
|
|
371
|
+
? { warning: "PDF contains little or no extractable text" }
|
|
372
|
+
: {}),
|
|
373
|
+
};
|
|
374
|
+
}
|
|
375
|
+
finally {
|
|
376
|
+
await parser.destroy();
|
|
377
|
+
}
|
|
378
|
+
}
|
|
379
|
+
export async function extractStructuredDocument(filePath, data, signal) {
|
|
380
|
+
const extension = path.extname(filePath).toLowerCase();
|
|
381
|
+
const legacyTarget = legacyOfficeTargets[extension];
|
|
382
|
+
if (legacyTarget) {
|
|
383
|
+
const converted = await runOfficeConversion(filePath, legacyTarget, signal);
|
|
384
|
+
try {
|
|
385
|
+
const extraction = await extractStructuredDocument(converted.path, await readFile(converted.path), signal);
|
|
386
|
+
return { ...extraction, documentType: extension.slice(1) };
|
|
387
|
+
}
|
|
388
|
+
finally {
|
|
389
|
+
await converted.cleanup();
|
|
390
|
+
}
|
|
391
|
+
}
|
|
392
|
+
if (extension === ".pdf") {
|
|
393
|
+
const extraction = await extractPdfDocument(data);
|
|
394
|
+
const limited = limitedContent(extraction.content);
|
|
395
|
+
return {
|
|
396
|
+
...limited,
|
|
397
|
+
documentType: "pdf",
|
|
398
|
+
totalPages: extraction.totalPages,
|
|
399
|
+
pagesParsed: extraction.pagesParsed,
|
|
400
|
+
truncated: extraction.truncated || limited.truncated,
|
|
401
|
+
...(extraction.warning ? { warning: extraction.warning } : {}),
|
|
402
|
+
};
|
|
403
|
+
}
|
|
404
|
+
let extracted;
|
|
405
|
+
switch (extension) {
|
|
406
|
+
case ".docx":
|
|
407
|
+
extracted = extractDocx(data);
|
|
408
|
+
break;
|
|
409
|
+
case ".pptx":
|
|
410
|
+
extracted = extractPptx(data);
|
|
411
|
+
break;
|
|
412
|
+
case ".xlsx":
|
|
413
|
+
extracted = extractXlsx(data);
|
|
414
|
+
break;
|
|
415
|
+
default:
|
|
416
|
+
throw new Error(`unsupported structured document: ${extension}`);
|
|
417
|
+
}
|
|
418
|
+
return {
|
|
419
|
+
...limitedContent(extracted),
|
|
420
|
+
documentType: extension.slice(1),
|
|
421
|
+
};
|
|
422
|
+
}
|
|
423
|
+
//# sourceMappingURL=document-reader.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"document-reader.js","sourceRoot":"","sources":["../src/document-reader.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,MAAM,oBAAoB,CAAC;AAC3C,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAE,EAAE,EAAE,MAAM,kBAAkB,CAAC;AACzE,OAAO,EAAE,MAAM,EAAE,MAAM,SAAS,CAAC;AACjC,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AAEzC,OAAO,EAAE,SAAS,EAAE,SAAS,EAAE,MAAM,QAAQ,CAAC;AAC9C,OAAO,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAC;AAErC,MAAM,uBAAuB,GAAG,EAAE,GAAG,IAAI,GAAG,IAAI,CAAC;AACjD,MAAM,uBAAuB,GAAG,EAAE,GAAG,IAAI,GAAG,IAAI,CAAC;AACjD,MAAM,wBAAwB,GAAG,OAAO,CAAC;AACzC,MAAM,qBAAqB,GAAG,OAAO,CAAC;AACtC,MAAM,4BAA4B,GAAG,MAAM,CAAC;AAE5C,MAAM,4BAA4B,GAAG,IAAI,GAAG,CAAC;IAC3C,MAAM;IACN,OAAO;IACP,MAAM;IACN,MAAM;IACN,OAAO;IACP,MAAM;IACN,OAAO;CACR,CAAC,CAAC;AAEH,MAAM,mBAAmB,GAA2B;IAClD,MAAM,EAAE,OAAO;IACf,MAAM,EAAE,OAAO;IACf,MAAM,EAAE,OAAO;CAChB,CAAC;AAmBF,SAAS,sBAAsB,CAAC,KAAa;IAC3C,OAAO,KAAK;SACT,OAAO,CAAC,QAAQ,EAAE,IAAI,CAAC;SACvB,OAAO,CAAC,aAAa,EAAE,IAAI,CAAC;SAC5B,OAAO,CAAC,SAAS,EAAE,MAAM,CAAC;SAC1B,IAAI,EAAE,CAAC;AACZ,CAAC;AAED,SAAS,cAAc,CACrB,KAAa;IAEb,MAAM,OAAO,GAAG,sBAAsB,CAAC,KAAK,CAAC,CAAC;IAC9C,IAAI,OAAO,CAAC,MAAM,IAAI,wBAAwB;QAAE,OAAO,EAAE,OAAO,EAAE,CAAC;IACnE,OAAO;QACL,OAAO,EACL,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,wBAAwB,CAAC;YAC1C,0CAA0C;QAC5C,SAAS,EAAE,IAAI;KAChB,CAAC;AACJ,CAAC;AAED,SAAS,iBAAiB,CAAC,KAAa;IACtC,OAAO,KAAK,CAAC,OAAO,CAClB,kDAAkD,EAClD,CAAC,MAAM,EAAE,OAA2B,EAAE,WAA+B,EAAE,EAAE;QACvE,IAAI,OAAO;YAAE,OAAO,MAAM,CAAC,aAAa,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC;QAC1D,IAAI,WAAW;YAAE,OAAO,MAAM,CAAC,aAAa,CAAC,MAAM,CAAC,QAAQ,CAAC,WAAW,EAAE,EAAE,CAAC,CAAC,CAAC;QAC/E,QAAQ,MAAM,CAAC,WAAW,EAAE,EAAE,CAAC;YAC7B,KAAK,OAAO;gBACV,OAAO,GAAG,CAAC;YACb,KAAK,QAAQ;gBACX,OAAO,GAAG,CAAC;YACb,KAAK,MAAM;gBACT,OAAO,GAAG,CAAC;YACb,KAAK,MAAM;gBACT,OAAO,GAAG,CAAC;YACb,KAAK,QAAQ;gBACX,OAAO,GAAG,CAAC;YACb;gBACE,OAAO,MAAM,CAAC;QAClB,CAAC;IACH,CAAC,CACF,CAAC;AACJ,CAAC;AAED,SAAS,YAAY,CAAC,GAAW,EAAE,IAAY;IAC7C,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,qBAAqB,EAAE,MAAM,CAAC,CAAC;IAC5D,MAAM,KAAK,GAAG,GAAG,CAAC,KAAK,CACrB,IAAI,MAAM,CAAC,YAAY,OAAO,kCAAkC,EAAE,GAAG,CAAC,CACvE,CAAC;IACF,OAAO,iBAAiB,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;AAC3D,CAAC;AAED,SAAS,OAAO,CAAC,KAAa;IAC5B,OAAO,iBAAiB,CAAC,KAAK,CAAC,OAAO,CAAC,UAAU,EAAE,EAAE,CAAC,CAAC,CAAC;AAC1D,CAAC;AAED,SAAS,cAAc,CACrB,IAAY,EACZ,OAAkC;IAElC,IAAI,aAAa,GAAG,CAAC,CAAC;IACtB,IAAI,CAAC;QACH,OAAO,SAAS,CAAC,IAAI,EAAE;YACrB,MAAM,CAAC,IAAI;gBACT,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;gBAC7C,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC;oBAAE,OAAO,KAAK,CAAC;gBACjC,IAAI,IAAI,CAAC,YAAY,GAAG,uBAAuB,EAAE,CAAC;oBAChD,MAAM,IAAI,KAAK,CAAC,+BAA+B,IAAI,EAAE,CAAC,CAAC;gBACzD,CAAC;gBACD,aAAa,IAAI,IAAI,CAAC,YAAY,CAAC;gBACnC,IAAI,aAAa,GAAG,uBAAuB,EAAE,CAAC;oBAC5C,MAAM,IAAI,KAAK,CAAC,mDAAmD,CAAC,CAAC;gBACvE,CAAC;gBACD,OAAO,IAAI,CAAC;YACd,CAAC;SACF,CAAC,CAAC;IACL,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,IAAI,KAAK,CAAC,qCAAqC,YAAY,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;IAC9E,CAAC;AACH,CAAC;AAED,SAAS,SAAS,CAChB,OAAmC,EACnC,IAAY;IAEZ,MAAM,KAAK,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAC5B,OAAO,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;AACvC,CAAC;AAED,SAAS,gBAAgB,CAAC,IAAY,EAAE,KAAa;IACnD,MAAM,UAAU,GAAG,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,iBAAiB,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;IACnE,MAAM,WAAW,GAAG,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,iBAAiB,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;IACrE,OAAO,UAAU,GAAG,WAAW,IAAI,IAAI,CAAC,aAAa,CAAC,KAAK,CAAC,CAAC;AAC/D,CAAC;AAED,SAAS,WAAW,CAAC,IAAY;IAC/B,MAAM,OAAO,GAAG,cAAc,CAC5B,IAAI,EACJ,CAAC,IAAI,EAAE,EAAE,CACP,IAAI,KAAK,mBAAmB;QAC5B,mCAAmC,CAAC,IAAI,CAAC,IAAI,CAAC;QAC9C,8CAA8C,CAAC,IAAI,CAAC,IAAI,CAAC,CAC5D,CAAC;IACF,IAAI,CAAC,OAAO,CAAC,mBAAmB,CAAC,EAAE,CAAC;QAClC,MAAM,IAAI,KAAK,CAAC,yCAAyC,CAAC,CAAC;IAC7D,CAAC;IAED,MAAM,WAAW,GAAG,CAAC,GAAW,EAAU,EAAE,CAC1C,sBAAsB,CACpB,iBAAiB,CACf,GAAG;SACA,OAAO,CAAC,oBAAoB,EAAE,IAAI,CAAC;SACnC,OAAO,CAAC,mBAAmB,EAAE,IAAI,CAAC;SAClC,OAAO,CAAC,YAAY,EAAE,IAAI,CAAC;SAC3B,OAAO,CAAC,kBAAkB,EAAE,IAAI,CAAC;SACjC,OAAO,CAAC,UAAU,EAAE,EAAE,CAAC,CAC3B,CACF,CAAC;IAEJ,MAAM,QAAQ,GAAa,EAAE,CAAC;IAC9B,MAAM,IAAI,GAAG,WAAW,CAAC,SAAS,CAAC,OAAO,EAAE,mBAAmB,CAAC,CAAC,CAAC;IAClE,IAAI,IAAI;QAAE,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC9B,KAAK,MAAM,IAAI,IAAI,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC;SACpC,MAAM,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,SAAS,KAAK,mBAAmB,CAAC;SACxD,IAAI,CAAC,gBAAgB,CAAC,EAAE,CAAC;QAC1B,MAAM,OAAO,GAAG,WAAW,CAAC,SAAS,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC,CAAC;QACtD,IAAI,CAAC,OAAO;YAAE,SAAS;QACvB,MAAM,KAAK,GAAG,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC,OAAO,CAAC,WAAW,EAAE,OAAO,CAAC,CAAC;QACxE,QAAQ,CAAC,IAAI,CAAC,MAAM,KAAK,OAAO,OAAO,EAAE,CAAC,CAAC;IAC7C,CAAC;IACD,OAAO,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,wBAAwB,CAAC;AAC3D,CAAC;AAED,SAAS,WAAW,CAAC,IAAY;IAC/B,MAAM,OAAO,GAAG,cAAc,CAC5B,IAAI,EACJ,CAAC,IAAI,EAAE,EAAE,CACP,8BAA8B,CAAC,IAAI,CAAC,IAAI,CAAC;QACzC,wCAAwC,CAAC,IAAI,CAAC,IAAI,CAAC,CACtD,CAAC;IACF,MAAM,MAAM,GAAG,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC;SAChC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,8BAA8B,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;SAC3D,IAAI,CAAC,gBAAgB,CAAC,CAAC;IAC1B,IAAI,CAAC,MAAM,CAAC,MAAM;QAAE,MAAM,IAAI,KAAK,CAAC,kCAAkC,CAAC,CAAC;IAExE,MAAM,WAAW,GAAG,CAAC,GAAW,EAAU,EAAE,CAC1C,sBAAsB,CACpB,iBAAiB,CACf,GAAG;SACA,OAAO,CAAC,oBAAoB,EAAE,IAAI,CAAC;SACnC,OAAO,CAAC,mBAAmB,EAAE,IAAI,CAAC;SAClC,OAAO,CAAC,WAAW,EAAE,IAAI,CAAC;SAC1B,OAAO,CAAC,UAAU,EAAE,EAAE,CAAC,CAC3B,CACF,CAAC;IAEJ,OAAO,MAAM;SACV,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;QACZ,MAAM,MAAM,GAAG,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,kBAAkB,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;QAChE,MAAM,OAAO,GAAG,WAAW,CAAC,SAAS,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC,CAAC;QACtD,MAAM,SAAS,GAAG,6BAA6B,MAAM,MAAM,CAAC;QAC5D,MAAM,KAAK,GAAG,WAAW,CAAC,SAAS,CAAC,OAAO,EAAE,SAAS,CAAC,CAAC,CAAC;QACzD,OAAO;YACL,YAAY,MAAM,EAAE;YACpB,OAAO,IAAI,wBAAwB;YACnC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,wBAAwB,KAAK,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;SACpD,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IACjB,CAAC,CAAC;SACD,IAAI,CAAC,MAAM,CAAC,CAAC;AAClB,CAAC;AAED,SAAS,iBAAiB,CAAC,SAAiB;IAC1C,MAAM,OAAO,GAAG,SAAS,CAAC,KAAK,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,IAAI,EAAE,CAAC;IACtE,IAAI,MAAM,GAAG,CAAC,CAAC;IACf,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;QAC7B,MAAM,GAAG,MAAM,GAAG,EAAE,GAAG,MAAM,CAAC,UAAU,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC;IACnD,CAAC;IACD,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,MAAM,GAAG,CAAC,CAAC,CAAC;AACjC,CAAC;AAED,SAAS,kBAAkB,CAAC,MAAc;IACxC,MAAM,UAAU,GAAG,MAAM,CAAC,UAAU,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;IACpE,IAAI,UAAU,CAAC,UAAU,CAAC,KAAK,CAAC;QAAE,OAAO,UAAU,CAAC;IACpD,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,KAAK,MAAM,IAAI,IAAI,MAAM,UAAU,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE,CAAC;QACjD,IAAI,IAAI,KAAK,IAAI;YAAE,KAAK,CAAC,GAAG,EAAE,CAAC;aAC1B,IAAI,IAAI,IAAI,IAAI,KAAK,GAAG;YAAE,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAClD,CAAC;IACD,OAAO,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;AACzB,CAAC;AAED,SAAS,WAAW,CAAC,IAAY;IAC/B,MAAM,OAAO,GAAG,cAAc,CAC5B,IAAI,EACJ,CAAC,IAAI,EAAE,EAAE,CACP,IAAI,KAAK,iBAAiB;QAC1B,IAAI,KAAK,4BAA4B;QACrC,IAAI,KAAK,sBAAsB;QAC/B,iCAAiC,CAAC,IAAI,CAAC,IAAI,CAAC,CAC/C,CAAC;IACF,MAAM,cAAc,GAAG,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC;SACxC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,iCAAiC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;SAC9D,IAAI,CAAC,gBAAgB,CAAC,CAAC;IAC1B,IAAI,CAAC,cAAc,CAAC,MAAM,EAAE,CAAC;QAC3B,MAAM,IAAI,KAAK,CAAC,sCAAsC,CAAC,CAAC;IAC1D,CAAC;IAED,MAAM,aAAa,GAAG;QACpB,GAAG,SAAS,CAAC,OAAO,EAAE,sBAAsB,CAAC,CAAC,QAAQ,CACpD,+BAA+B,CAChC;KACF,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;IAE1C,MAAM,aAAa,GAAG,IAAI,GAAG,EAAkB,CAAC;IAChD,KAAK,MAAM,KAAK,IAAI,SAAS,CAC3B,OAAO,EACP,4BAA4B,CAC7B,CAAC,QAAQ,CAAC,8BAA8B,CAAC,EAAE,CAAC;QAC3C,MAAM,EAAE,GAAG,YAAY,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,EAAE,EAAE,IAAI,CAAC,CAAC;QAC9C,MAAM,MAAM,GAAG,YAAY,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,EAAE,EAAE,QAAQ,CAAC,CAAC;QACtD,IAAI,EAAE,IAAI,MAAM;YAAE,aAAa,CAAC,GAAG,CAAC,EAAE,EAAE,kBAAkB,CAAC,MAAM,CAAC,CAAC,CAAC;IACtE,CAAC;IAED,MAAM,UAAU,GAAG,IAAI,GAAG,EAAkB,CAAC;IAC7C,KAAK,MAAM,KAAK,IAAI,SAAS,CAAC,OAAO,EAAE,iBAAiB,CAAC,CAAC,QAAQ,CAChE,uBAAuB,CACxB,EAAE,CAAC;QACF,MAAM,UAAU,GAAG,KAAK,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;QAClC,MAAM,EAAE,GAAG,YAAY,CAAC,UAAU,EAAE,MAAM,CAAC,CAAC;QAC5C,MAAM,IAAI,GAAG,YAAY,CAAC,UAAU,EAAE,MAAM,CAAC,CAAC;QAC9C,MAAM,MAAM,GAAG,aAAa,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QACrC,IAAI,MAAM,IAAI,IAAI;YAAE,UAAU,CAAC,GAAG,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;IACnD,CAAC;IAED,IAAI,SAAS,GAAG,CAAC,CAAC;IAClB,OAAO,cAAc;SAClB,GAAG,CAAC,CAAC,aAAa,EAAE,UAAU,EAAE,EAAE;QACjC,MAAM,IAAI,GAAG,IAAI,GAAG,EAA+B,CAAC;QACpD,MAAM,GAAG,GAAG,SAAS,CAAC,OAAO,EAAE,aAAa,CAAC,CAAC;QAC9C,KAAK,MAAM,KAAK,IAAI,GAAG,CAAC,QAAQ,CAAC,+BAA+B,CAAC,EAAE,CAAC;YAClE,SAAS,IAAI,CAAC,CAAC;YACf,IAAI,SAAS,GAAG,qBAAqB,EAAE,CAAC;gBACtC,MAAM,IAAI,KAAK,CACb,uBAAuB,qBAAqB,iBAAiB,CAC9D,CAAC;YACJ,CAAC;YACD,MAAM,UAAU,GAAG,KAAK,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;YAClC,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;YAC5B,MAAM,SAAS,GAAG,YAAY,CAAC,UAAU,EAAE,GAAG,CAAC,CAAC;YAChD,MAAM,IAAI,GAAG,YAAY,CAAC,UAAU,EAAE,GAAG,CAAC,CAAC;YAC3C,MAAM,SAAS,GAAG,MAAM,CAAC,SAAS,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;YAC5D,MAAM,MAAM,GAAG,iBAAiB,CAAC,SAAS,CAAC,CAAC;YAC5C,MAAM,QAAQ,GAAG,iBAAiB,CAChC,IAAI,CAAC,KAAK,CAAC,4BAA4B,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CACpD,CAAC;YACF,MAAM,OAAO,GAAG,iBAAiB,CAC/B,IAAI,CAAC,KAAK,CAAC,4BAA4B,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CACpD,CAAC;YACF,IAAI,KAAa,CAAC;YAClB,IAAI,IAAI,KAAK,GAAG,EAAE,CAAC;gBACjB,KAAK,GAAG,aAAa,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,IAAI,EAAE,CAAC;YAChD,CAAC;iBAAM,IAAI,IAAI,KAAK,WAAW,EAAE,CAAC;gBAChC,KAAK,GAAG,OAAO,CACb,IAAI,CAAC,KAAK,CAAC,8BAA8B,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CACtD,CAAC;YACJ,CAAC;iBAAM,IAAI,IAAI,KAAK,GAAG,EAAE,CAAC;gBACxB,KAAK,GAAG,QAAQ,KAAK,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC;YAC9C,CAAC;iBAAM,CAAC;gBACN,KAAK,GAAG,QAAQ,CAAC;YACnB,CAAC;YACD,IAAI,OAAO,EAAE,CAAC;gBACZ,KAAK,GAAG,KAAK,CAAC,CAAC,CAAC,IAAI,OAAO,OAAO,KAAK,EAAE,CAAC,CAAC,CAAC,IAAI,OAAO,EAAE,CAAC;YAC5D,CAAC;YACD,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,IAAI,IAAI,GAAG,EAAkB,CAAC;YAC7D,GAAG,CAAC,GAAG,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;YACvB,IAAI,CAAC,GAAG,CAAC,SAAS,EAAE,GAAG,CAAC,CAAC;QAC3B,CAAC;QAED,MAAM,KAAK,GAAG,CAAC,GAAG,IAAI,CAAC,OAAO,EAAE,CAAC;aAC9B,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,KAAK,CAAC,EAAE,EAAE,CAAC,IAAI,GAAG,KAAK,CAAC;aACvC,GAAG,CAAC,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,EAAE;YACf,MAAM,aAAa,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,GAAG,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,CAAC;YACjD,OAAO,KAAK,CAAC,IAAI,CACf,EAAE,MAAM,EAAE,aAAa,GAAG,CAAC,EAAE,EAC7B,CAAC,CAAC,EAAE,MAAM,EAAE,EAAE,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,EAAE,CACrC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACf,CAAC,CAAC,CAAC;QACL,MAAM,IAAI,GAAG,UAAU,CAAC,GAAG,CAAC,aAAa,CAAC,IAAI,SAAS,UAAU,GAAG,CAAC,EAAE,CAAC;QACxE,OAAO,aAAa,IAAI,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,uBAAuB,EAAE,CAAC;IAC/E,CAAC,CAAC;SACD,IAAI,CAAC,MAAM,CAAC,CAAC;AAClB,CAAC;AAED,SAAS,YAAY,CAAC,KAAc;IAClC,OAAO,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;AAChE,CAAC;AAED,KAAK,UAAU,mBAAmB,CAChC,UAAkB,EAClB,eAAuB,EACvB,MAAoB;IAEpB,MAAM,kBAAkB,GAAG,MAAM,OAAO,CACtC,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,EAAE,kBAAkB,CAAC,CACxC,CAAC;IACF,MAAM,eAAe,GAAG,IAAI,CAAC,IAAI,CAAC,kBAAkB,EAAE,QAAQ,CAAC,CAAC;IAChE,MAAM,gBAAgB,GAAG,IAAI,CAAC,IAAI,CAAC,kBAAkB,EAAE,SAAS,CAAC,CAAC;IAClE,MAAM,OAAO,CAAC,GAAG,CAAC;QAChB,KAAK,CAAC,eAAe,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC;QAC3C,KAAK,CAAC,gBAAgB,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC;KAC7C,CAAC,CAAC;IACH,MAAM,OAAO,GAAG,OAAO,CAAC,GAAG,CAAC,kBAAkB,IAAI,SAAS,CAAC;IAC5D,MAAM,UAAU,GAAG;QACjB,YAAY;QACZ,UAAU;QACV,eAAe;QACf,aAAa;QACb,sBAAsB;QACtB,yBAAyB,aAAa,CAAC,gBAAgB,CAAC,CAAC,IAAI,EAAE;QAC/D,cAAc;QACd,eAAe,CAAC,KAAK,CAAC,CAAC,CAAC;QACxB,UAAU;QACV,eAAe;QACf,UAAU;KACX,CAAC;IAEF,IAAI,CAAC;QACH,MAAM,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YAC1C,MAAM,KAAK,GAAG,KAAK,CAAC,OAAO,EAAE,UAAU,EAAE;gBACvC,KAAK,EAAE,CAAC,QAAQ,EAAE,MAAM,EAAE,MAAM,CAAC;aAClC,CAAC,CAAC;YACH,IAAI,MAAM,GAAG,EAAE,CAAC;YAChB,IAAI,QAAQ,GAAG,KAAK,CAAC;YACrB,MAAM,OAAO,GAAG,UAAU,CAAC,GAAG,EAAE;gBAC9B,QAAQ,GAAG,IAAI,CAAC;gBAChB,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;YACxB,CAAC,EAAE,4BAA4B,CAAC,CAAC;YACjC,MAAM,KAAK,GAAG,GAAG,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;YAC1C,MAAM,EAAE,gBAAgB,CAAC,OAAO,EAAE,KAAK,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC;YACzD,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,KAAa,EAAE,EAAE;gBACxC,IAAI,MAAM,CAAC,MAAM,GAAG,MAAM;oBAAE,MAAM,IAAI,KAAK,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;YAC/D,CAAC,CAAC,CAAC;YACH,KAAK,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC,KAAK,EAAE,EAAE;gBAC5B,YAAY,CAAC,OAAO,CAAC,CAAC;gBACtB,MAAM,EAAE,mBAAmB,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC;gBAC5C,IAAK,KAA+B,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;oBACvD,MAAM,CACJ,IAAI,KAAK,CACP,uDAAuD;wBACrD,sCAAsC,CACzC,CACF,CAAC;gBACJ,CAAC;qBAAM,CAAC;oBACN,MAAM,CAAC,KAAK,CAAC,CAAC;gBAChB,CAAC;YACH,CAAC,CAAC,CAAC;YACH,KAAK,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC,IAAI,EAAE,EAAE;gBAC3B,YAAY,CAAC,OAAO,CAAC,CAAC;gBACtB,MAAM,EAAE,mBAAmB,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC;gBAC5C,IAAI,MAAM,EAAE,OAAO,EAAE,CAAC;oBACpB,MAAM,CAAC,MAAM,CAAC,MAAM,IAAI,IAAI,KAAK,CAAC,qBAAqB,CAAC,CAAC,CAAC;gBAC5D,CAAC;qBAAM,IAAI,QAAQ,EAAE,CAAC;oBACpB,MAAM,CAAC,IAAI,KAAK,CAAC,kCAAkC,CAAC,CAAC,CAAC;gBACxD,CAAC;qBAAM,IAAI,IAAI,KAAK,CAAC,EAAE,CAAC;oBACtB,MAAM,CACJ,IAAI,KAAK,CACP,kCAAkC,IAAI,MAAM,MAAM,CAAC,IAAI,EAAE,EAAE,CAC5D,CACF,CAAC;gBACJ,CAAC;qBAAM,CAAC;oBACN,OAAO,EAAE,CAAC;gBACZ,CAAC;YACH,CAAC,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;QAEH,MAAM,KAAK,GAAG,MAAM,OAAO,CAAC,eAAe,CAAC,CAAC;QAC7C,MAAM,aAAa,GAAG,KAAK,CAAC,IAAI,CAC9B,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,WAAW,EAAE,KAAK,eAAe,CAC/D,CAAC;QACF,IAAI,CAAC,aAAa,EAAE,CAAC;YACnB,MAAM,IAAI,KAAK,CAAC,kDAAkD,CAAC,CAAC;QACtE,CAAC;QACD,OAAO;YACL,IAAI,EAAE,IAAI,CAAC,IAAI,CAAC,eAAe,EAAE,aAAa,CAAC;YAC/C,OAAO,EAAE,GAAG,EAAE,CAAC,EAAE,CAAC,kBAAkB,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC;SACxE,CAAC;IACJ,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,EAAE,CAAC,kBAAkB,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;QAC/D,MAAM,KAAK,CAAC;IACd,CAAC;AACH,CAAC;AAED,MAAM,UAAU,oBAAoB,CAAC,QAAgB;IACnD,OAAO,4BAA4B,CAAC,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC;AAChF,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,kBAAkB,CACtC,IAAgB,EAChB,UAAU,GAAG,EAAE;IAEf,MAAM,MAAM,GAAG,IAAI,QAAQ,CAAC,EAAE,IAAI,EAAE,IAAI,UAAU,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAC5D,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,OAAO,CAAC;YAClC,KAAK,EAAE,UAAU;YACjB,eAAe,EAAE,IAAI;YACrB,UAAU,EAAE,EAAE;SACf,CAAC,CAAC;QACH,MAAM,OAAO,GAAG,sBAAsB,CACpC,MAAM,CAAC,KAAK;aACT,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;YACZ,MAAM,IAAI,GAAG,sBAAsB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAC/C,OAAO,WAAW,IAAI,CAAC,GAAG,OAAO,IAAI,IAAI,wBAAwB,EAAE,CAAC;QACtE,CAAC,CAAC;aACD,IAAI,CAAC,MAAM,CAAC,CAChB,CAAC;QACF,MAAM,WAAW,GAAG,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC;QACxC,MAAM,kBAAkB,GAAG,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC;QACzE,OAAO;YACL,OAAO;YACP,UAAU,EAAE,MAAM,CAAC,KAAK;YACxB,WAAW;YACX,SAAS,EAAE,MAAM,CAAC,KAAK,GAAG,WAAW;YACrC,GAAG,CAAC,CAAC,kBAAkB;gBACrB,CAAC,CAAC,EAAE,OAAO,EAAE,4CAA4C,EAAE;gBAC3D,CAAC,CAAC,EAAE,CAAC;SACR,CAAC;IACJ,CAAC;YAAS,CAAC;QACT,MAAM,MAAM,CAAC,OAAO,EAAE,CAAC;IACzB,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,yBAAyB,CAC7C,QAAgB,EAChB,IAAY,EACZ,MAAoB;IAEpB,MAAM,SAAS,GAAG,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,WAAW,EAAE,CAAC;IACvD,MAAM,YAAY,GAAG,mBAAmB,CAAC,SAAS,CAAC,CAAC;IACpD,IAAI,YAAY,EAAE,CAAC;QACjB,MAAM,SAAS,GAAG,MAAM,mBAAmB,CAAC,QAAQ,EAAE,YAAY,EAAE,MAAM,CAAC,CAAC;QAC5E,IAAI,CAAC;YACH,MAAM,UAAU,GAAG,MAAM,yBAAyB,CAChD,SAAS,CAAC,IAAI,EACd,MAAM,QAAQ,CAAC,SAAS,CAAC,IAAI,CAAC,EAC9B,MAAM,CACP,CAAC;YACF,OAAO,EAAE,GAAG,UAAU,EAAE,YAAY,EAAE,SAAS,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC;QAC7D,CAAC;gBAAS,CAAC;YACT,MAAM,SAAS,CAAC,OAAO,EAAE,CAAC;QAC5B,CAAC;IACH,CAAC;IAED,IAAI,SAAS,KAAK,MAAM,EAAE,CAAC;QACzB,MAAM,UAAU,GAAG,MAAM,kBAAkB,CAAC,IAAI,CAAC,CAAC;QAClD,MAAM,OAAO,GAAG,cAAc,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC;QACnD,OAAO;YACL,GAAG,OAAO;YACV,YAAY,EAAE,KAAK;YACnB,UAAU,EAAE,UAAU,CAAC,UAAU;YACjC,WAAW,EAAE,UAAU,CAAC,WAAW;YACnC,SAAS,EAAE,UAAU,CAAC,SAAS,IAAI,OAAO,CAAC,SAAS;YACpD,GAAG,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,UAAU,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SAC/D,CAAC;IACJ,CAAC;IAED,IAAI,SAAiB,CAAC;IACtB,QAAQ,SAAS,EAAE,CAAC;QAClB,KAAK,OAAO;YACV,SAAS,GAAG,WAAW,CAAC,IAAI,CAAC,CAAC;YAC9B,MAAM;QACR,KAAK,OAAO;YACV,SAAS,GAAG,WAAW,CAAC,IAAI,CAAC,CAAC;YAC9B,MAAM;QACR,KAAK,OAAO;YACV,SAAS,GAAG,WAAW,CAAC,IAAI,CAAC,CAAC;YAC9B,MAAM;QACR;YACE,MAAM,IAAI,KAAK,CAAC,oCAAoC,SAAS,EAAE,CAAC,CAAC;IACrE,CAAC;IACD,OAAO;QACL,GAAG,cAAc,CAAC,SAAS,CAAC;QAC5B,YAAY,EAAE,SAAS,CAAC,KAAK,CAAC,CAAC,CAAC;KACjC,CAAC;AACJ,CAAC"}
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
type LookupAddress = {
|
|
2
|
+
address: string;
|
|
3
|
+
family: number;
|
|
4
|
+
};
|
|
5
|
+
type LookupFunction = (hostname: string) => Promise<LookupAddress[]>;
|
|
6
|
+
interface FetchPageReaderOptions {
|
|
7
|
+
signal?: AbortSignal;
|
|
8
|
+
fetchImpl?: typeof fetch;
|
|
9
|
+
lookupImpl?: LookupFunction;
|
|
10
|
+
chunkSize?: number;
|
|
11
|
+
timeoutMs?: number;
|
|
12
|
+
}
|
|
13
|
+
export declare class FetchPageReader {
|
|
14
|
+
private readonly fetchImpl;
|
|
15
|
+
private readonly lookupImpl;
|
|
16
|
+
private readonly chunkSize;
|
|
17
|
+
private readonly timeoutMs;
|
|
18
|
+
private readonly signal?;
|
|
19
|
+
private readonly cache;
|
|
20
|
+
constructor(options?: FetchPageReaderOptions);
|
|
21
|
+
read(url: string, chunkIndex: number): Promise<Record<string, unknown>>;
|
|
22
|
+
private load;
|
|
23
|
+
private download;
|
|
24
|
+
}
|
|
25
|
+
export {};
|