pagerts 1.5.2 โ 1.5.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +40 -9
- package/bin/main.js +167 -82
- package/bin/main.js.map +2 -2
- package/package.json +5 -4
package/README.md
CHANGED
|
@@ -14,6 +14,8 @@ PagerTS is a secure, modern command-line utility that transforms URLs into struc
|
|
|
14
14
|
- โก **Fast**: Efficient parsing with LinkeDOM and concurrent request handling
|
|
15
15
|
- ๐งช **Well-Tested**: Comprehensive test coverage with Jest
|
|
16
16
|
- ๐ฆ **Easy to Use**: Simple CLI interface with sensible defaults
|
|
17
|
+
- ๐๏ธ **Local File Support**: bare `pagerts` parses local file paths and `file:///...` inputs
|
|
18
|
+
- ๐งญ **Request Header Override**: Optional `--user-agent` flag for sites that behave differently by client
|
|
17
19
|
|
|
18
20
|
## Installation
|
|
19
21
|
|
|
@@ -21,13 +23,13 @@ PagerTS is a secure, modern command-line utility that transforms URLs into struc
|
|
|
21
23
|
|
|
22
24
|
```bash
|
|
23
25
|
npm install -g pagerts
|
|
24
|
-
pagerts
|
|
26
|
+
pagerts ./page.html
|
|
25
27
|
```
|
|
26
28
|
|
|
27
29
|
### Using npx (No Installation Required)
|
|
28
30
|
|
|
29
31
|
```bash
|
|
30
|
-
npx pagerts
|
|
32
|
+
npx pagerts ./page.html
|
|
31
33
|
```
|
|
32
34
|
|
|
33
35
|
### From Source
|
|
@@ -44,22 +46,40 @@ npm link
|
|
|
44
46
|
|
|
45
47
|
### Basic Usage
|
|
46
48
|
|
|
47
|
-
Extract resources from a
|
|
49
|
+
Extract resources from a local HTML file path:
|
|
48
50
|
|
|
49
51
|
```bash
|
|
50
|
-
pagerts
|
|
52
|
+
pagerts ./page.html
|
|
51
53
|
```
|
|
52
54
|
|
|
53
|
-
Extract from
|
|
55
|
+
Extract resources from a local file URL:
|
|
54
56
|
|
|
55
57
|
```bash
|
|
56
|
-
pagerts
|
|
58
|
+
pagerts file:///path/to/file.html
|
|
57
59
|
```
|
|
58
60
|
|
|
59
|
-
|
|
61
|
+
Fetch resources from a remote URL:
|
|
60
62
|
|
|
61
63
|
```bash
|
|
62
|
-
pagerts
|
|
64
|
+
pagerts fetch https://website.com
|
|
65
|
+
```
|
|
66
|
+
|
|
67
|
+
Override the HTTP user-agent for remote fetches:
|
|
68
|
+
|
|
69
|
+
```bash
|
|
70
|
+
pagerts fetch --user-agent "Mozilla/5.0 (X11; Linux x86_64; rv:139.0) Gecko/20100101 Firefox/139.0" https://example.com
|
|
71
|
+
```
|
|
72
|
+
|
|
73
|
+
Allow fetching from localhost/private-network targets (opt-in):
|
|
74
|
+
|
|
75
|
+
```bash
|
|
76
|
+
pagerts fetch --allow-private-hosts http://127.0.0.1:3000
|
|
77
|
+
```
|
|
78
|
+
|
|
79
|
+
Fetch from multiple remote URLs:
|
|
80
|
+
|
|
81
|
+
```bash
|
|
82
|
+
pagerts fetch https://example.com https://example.org
|
|
63
83
|
```
|
|
64
84
|
|
|
65
85
|
### Output Format
|
|
@@ -98,10 +118,14 @@ PagerTS takes security seriously. See [SECURITY.md](./SECURITY.md) for:
|
|
|
98
118
|
|
|
99
119
|
### Built-in Security Features
|
|
100
120
|
|
|
101
|
-
- โ
URL validation (only allows `http
|
|
121
|
+
- โ
URL validation for remote fetches (only allows `http://` and `https://`)
|
|
122
|
+
- โ
Private/loopback hosts are blocked by default for remote fetches (SSRF mitigation)
|
|
123
|
+
- โ
Local filesystem parsing through plain paths and `file://` inputs on the root command
|
|
102
124
|
- โ
Input sanitization to prevent XSS attacks
|
|
103
125
|
- โ
Rate limiting (50 requests/minute by default)
|
|
104
126
|
- โ
Request timeouts to prevent hanging
|
|
127
|
+
- โ
HTTP status and content-type checks for remote responses
|
|
128
|
+
- โ
Maximum remote HTML response size enforcement (2 MiB)
|
|
105
129
|
- โ
Maximum URL length enforcement
|
|
106
130
|
- โ
Suspicious pattern detection
|
|
107
131
|
- โ
Safe HTML parsing (no script execution)
|
|
@@ -226,6 +250,13 @@ This project is licensed under the MIT License - see the [LICENSE](./LICENSE) fi
|
|
|
226
250
|
|
|
227
251
|
## Changelog
|
|
228
252
|
|
|
253
|
+
### v1.5.3
|
|
254
|
+
|
|
255
|
+
- Added `--user-agent` support to the `fetch` command so callers can override the HTTP User-Agent header for remote requests.
|
|
256
|
+
- Made the root `pagerts` command parse local file paths and `file:///` inputs directly, while keeping `fetch` remote-only.
|
|
257
|
+
- Improved CLI/runtime compatibility for locally resolved entrypoints and packaged builds.
|
|
258
|
+
- Updated focused tests to cover the new file-protocol validation and user-agent override behavior.
|
|
259
|
+
|
|
229
260
|
### v0.3.0 -> v1.4.3 summary
|
|
230
261
|
|
|
231
262
|
Key changes in this range:
|
package/bin/main.js
CHANGED
|
@@ -12,6 +12,7 @@ var AbstractExtractor = class {
|
|
|
12
12
|
constructor(name2) {
|
|
13
13
|
this.name = name2;
|
|
14
14
|
}
|
|
15
|
+
name;
|
|
15
16
|
};
|
|
16
17
|
|
|
17
18
|
// src/extractors/PageExtractor.ts
|
|
@@ -64,6 +65,7 @@ var ResourceExtractor = class extends AbstractExtractor {
|
|
|
64
65
|
super("page-extractor");
|
|
65
66
|
this.tags = tags;
|
|
66
67
|
}
|
|
68
|
+
tags;
|
|
67
69
|
async extract(value) {
|
|
68
70
|
const { document } = value.window;
|
|
69
71
|
return this.tags.flatMap(
|
|
@@ -79,12 +81,16 @@ var ResourceExtractor = class extends AbstractExtractor {
|
|
|
79
81
|
|
|
80
82
|
// src/page/PageFetcher.ts
|
|
81
83
|
import { parseHTML } from "linkedom";
|
|
84
|
+
var MAX_HTML_BYTES = 2 * 1024 * 1024;
|
|
85
|
+
var ALLOWED_CONTENT_TYPES = ["text/html", "application/xhtml+xml"];
|
|
82
86
|
var PageFetcher = class {
|
|
83
87
|
timeout;
|
|
84
88
|
maxRetries;
|
|
85
|
-
|
|
89
|
+
userAgent;
|
|
90
|
+
constructor(timeout = 1e4, maxRetries = 2, userAgent) {
|
|
86
91
|
this.timeout = timeout;
|
|
87
92
|
this.maxRetries = maxRetries;
|
|
93
|
+
this.userAgent = userAgent;
|
|
88
94
|
}
|
|
89
95
|
buildDOMResult(html, url) {
|
|
90
96
|
const { document } = parseHTML(html);
|
|
@@ -106,13 +112,33 @@ var PageFetcher = class {
|
|
|
106
112
|
controller.abort(new Error("Request timeout"));
|
|
107
113
|
}, this.timeout);
|
|
108
114
|
}
|
|
109
|
-
const
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
115
|
+
const headers = this.userAgent ? { "user-agent": this.userAgent } : void 0;
|
|
116
|
+
const content = await fetch(url, { headers, signal: controller.signal }).then(
|
|
117
|
+
async (response) => {
|
|
118
|
+
if (!response.ok) {
|
|
119
|
+
throw new Error(`HTTP ${response.status} ${response.statusText}`.trim());
|
|
120
|
+
}
|
|
121
|
+
const contentType = (response.headers.get("content-type") ?? "").toLowerCase();
|
|
122
|
+
const isAllowedContentType = ALLOWED_CONTENT_TYPES.some(
|
|
123
|
+
(allowedType) => contentType.includes(allowedType)
|
|
124
|
+
);
|
|
125
|
+
if (!isAllowedContentType) {
|
|
126
|
+
throw new Error(`Unsupported content type: ${contentType || "unknown"}`);
|
|
127
|
+
}
|
|
128
|
+
const contentLengthHeader = response.headers.get("content-length");
|
|
129
|
+
const contentLength = contentLengthHeader ? Number(contentLengthHeader) : Number.NaN;
|
|
130
|
+
if (Number.isFinite(contentLength) && contentLength > MAX_HTML_BYTES) {
|
|
131
|
+
throw new Error(`Response exceeds max allowed size (${MAX_HTML_BYTES} bytes)`);
|
|
132
|
+
}
|
|
133
|
+
const buffer = await response.arrayBuffer();
|
|
134
|
+
if (buffer.byteLength > MAX_HTML_BYTES) {
|
|
135
|
+
throw new Error(`Response exceeds max allowed size (${MAX_HTML_BYTES} bytes)`);
|
|
136
|
+
}
|
|
137
|
+
const charsetMatch = /charset=([^\s;]+)/i.exec(contentType);
|
|
138
|
+
const html = this.decodeHtml(buffer, charsetMatch?.[1] ?? "utf-8");
|
|
139
|
+
return this.buildDOMResult(html, url);
|
|
140
|
+
}
|
|
141
|
+
);
|
|
116
142
|
return { url, content };
|
|
117
143
|
} catch (error) {
|
|
118
144
|
const abortTimeout = error instanceof Error && error.name === "AbortError";
|
|
@@ -183,6 +209,7 @@ var JSONStylePrinter = class extends AbstractResourcePrinter {
|
|
|
183
209
|
};
|
|
184
210
|
|
|
185
211
|
// src/security.ts
|
|
212
|
+
import { isIP } from "node:net";
|
|
186
213
|
var ALLOWED_PROTOCOLS = ["http:", "https:"];
|
|
187
214
|
var MAX_URL_LENGTH = 2048;
|
|
188
215
|
var SUSPICIOUS_PATTERNS = [
|
|
@@ -193,7 +220,26 @@ var SUSPICIOUS_PATTERNS = [
|
|
|
193
220
|
/on\w+=/i
|
|
194
221
|
// Event handlers like onclick=
|
|
195
222
|
];
|
|
196
|
-
function
|
|
223
|
+
function isPrivateHostname(hostname) {
|
|
224
|
+
const normalized = hostname.toLowerCase();
|
|
225
|
+
if (normalized === "localhost" || normalized.endsWith(".localhost")) {
|
|
226
|
+
return true;
|
|
227
|
+
}
|
|
228
|
+
const ipType = isIP(normalized);
|
|
229
|
+
if (ipType === 0) {
|
|
230
|
+
return false;
|
|
231
|
+
}
|
|
232
|
+
if (ipType === 4) {
|
|
233
|
+
const octets = normalized.split(".").map(Number);
|
|
234
|
+
if (octets.length !== 4 || octets.some((octet) => Number.isNaN(octet))) {
|
|
235
|
+
return false;
|
|
236
|
+
}
|
|
237
|
+
const [a, b] = octets;
|
|
238
|
+
return a === 10 || a === 127 || a === 169 && b === 254 || a === 172 && b >= 16 && b <= 31 || a === 192 && b === 168 || a === 0;
|
|
239
|
+
}
|
|
240
|
+
return normalized === "::1" || normalized === "::" || normalized.startsWith("fc") || normalized.startsWith("fd") || normalized.startsWith("fe8") || normalized.startsWith("fe9") || normalized.startsWith("fea") || normalized.startsWith("feb");
|
|
241
|
+
}
|
|
242
|
+
function validateUrl(url, options = {}) {
|
|
197
243
|
if (!url || !url.trim()) {
|
|
198
244
|
return {
|
|
199
245
|
isValid: false,
|
|
@@ -230,21 +276,28 @@ function validateUrl(url) {
|
|
|
230
276
|
error: `Protocol ${parsedUrl.protocol} is not allowed. Allowed protocols: ${ALLOWED_PROTOCOLS.join(", ")}`
|
|
231
277
|
};
|
|
232
278
|
}
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
279
|
+
if (parsedUrl.username || parsedUrl.password) {
|
|
280
|
+
return {
|
|
281
|
+
isValid: false,
|
|
282
|
+
error: "URLs with embedded credentials are not allowed"
|
|
283
|
+
};
|
|
284
|
+
}
|
|
285
|
+
if (!options.allowPrivateHosts && isPrivateHostname(parsedUrl.hostname)) {
|
|
286
|
+
return {
|
|
287
|
+
isValid: false,
|
|
288
|
+
error: "Private or loopback hostnames are blocked by default. Use --allow-private-hosts if you trust the target."
|
|
289
|
+
};
|
|
237
290
|
}
|
|
238
291
|
return {
|
|
239
292
|
isValid: true,
|
|
240
293
|
sanitizedUrl: parsedUrl.toString()
|
|
241
294
|
};
|
|
242
295
|
}
|
|
243
|
-
function validateUrls(urls) {
|
|
296
|
+
function validateUrls(urls, options = {}) {
|
|
244
297
|
const validUrls = [];
|
|
245
298
|
const errors = [];
|
|
246
299
|
for (const url of urls) {
|
|
247
|
-
const result = validateUrl(url);
|
|
300
|
+
const result = validateUrl(url, options);
|
|
248
301
|
if (result.isValid && result.sanitizedUrl) {
|
|
249
302
|
validUrls.push(result.sanitizedUrl);
|
|
250
303
|
} else {
|
|
@@ -277,6 +330,37 @@ async function buildPageMetadata(responses) {
|
|
|
277
330
|
}
|
|
278
331
|
return pageMetadatas;
|
|
279
332
|
}
|
|
333
|
+
function normalizeLocalPath(value) {
|
|
334
|
+
if (value.startsWith("file://")) {
|
|
335
|
+
return fileURLToPath(value);
|
|
336
|
+
}
|
|
337
|
+
return value;
|
|
338
|
+
}
|
|
339
|
+
async function runFileCommand(paths, options) {
|
|
340
|
+
if (options.failsafe && paths.length > MAX_FILES_FAILSAFE) {
|
|
341
|
+
console.error(
|
|
342
|
+
`
|
|
343
|
+
\u274C ${paths.length} files specified exceeds the safety limit of ${MAX_FILES_FAILSAFE}.`
|
|
344
|
+
);
|
|
345
|
+
console.error(` Pass --no-failsafe to bypass this check and process all files.`);
|
|
346
|
+
process.exit(1);
|
|
347
|
+
}
|
|
348
|
+
if (!options.failsafe && paths.length > MAX_FILES_FAILSAFE) {
|
|
349
|
+
console.error(
|
|
350
|
+
`
|
|
351
|
+
\u26A0\uFE0F Failsafe bypassed: processing ${paths.length} files (limit is ${MAX_FILES_FAILSAFE}).`
|
|
352
|
+
);
|
|
353
|
+
}
|
|
354
|
+
console.error(`
|
|
355
|
+
\u2705 Processing ${paths.length} file(s)...`);
|
|
356
|
+
const fileFetcher = new FileFetcher();
|
|
357
|
+
const normalizedPaths = paths.map((pathValue) => normalizeLocalPath(pathValue));
|
|
358
|
+
const responses = await fileFetcher.fetchAll(normalizedPaths);
|
|
359
|
+
const pageMetadatas = await buildPageMetadata(
|
|
360
|
+
responses.map(({ path, content, error }) => ({ path, content, error }))
|
|
361
|
+
);
|
|
362
|
+
await printer.print(...pageMetadatas);
|
|
363
|
+
}
|
|
280
364
|
function isCliEntrypoint() {
|
|
281
365
|
const invokedPath = process.argv[1];
|
|
282
366
|
if (!invokedPath) {
|
|
@@ -290,85 +374,86 @@ function isCliEntrypoint() {
|
|
|
290
374
|
}
|
|
291
375
|
async function runCli(argv = process.argv) {
|
|
292
376
|
program.name(name).version(version, "-v, --version").description(description);
|
|
293
|
-
program.
|
|
377
|
+
program.addArgument(fileArg).addOption(
|
|
378
|
+
new Option("--no-failsafe", `bypass the ${MAX_FILES_FAILSAFE}-file limit safety check`)
|
|
379
|
+
).action(async (paths, options) => {
|
|
380
|
+
try {
|
|
381
|
+
await runFileCommand(paths, options);
|
|
382
|
+
} catch (error) {
|
|
383
|
+
console.error("\n\u274C An error occurred:", error instanceof Error ? error.message : error);
|
|
384
|
+
process.exit(1);
|
|
385
|
+
}
|
|
386
|
+
});
|
|
387
|
+
program.command("fetch").description("fetch and extract resources from remote URL(s)").addArgument(urlArg).addOption(
|
|
294
388
|
new Option(
|
|
295
389
|
"--watch",
|
|
296
390
|
"keep running: SIGWINCH re-fetches after resize, Ctrl-D releases in-flight requests, Ctrl-C exits"
|
|
297
391
|
)
|
|
298
|
-
).
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
392
|
+
).addOption(new Option("-A, --user-agent <value>", "override the HTTP User-Agent header")).addOption(
|
|
393
|
+
new Option(
|
|
394
|
+
"--allow-private-hosts",
|
|
395
|
+
"allow localhost/private-network targets (disabled by default for SSRF safety)"
|
|
396
|
+
)
|
|
397
|
+
).action(
|
|
398
|
+
async (urls, options) => {
|
|
399
|
+
try {
|
|
400
|
+
const { validUrls, errors } = validateUrls(urls, {
|
|
401
|
+
allowPrivateHosts: options.allowPrivateHosts
|
|
305
402
|
});
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
403
|
+
if (errors.length > 0) {
|
|
404
|
+
console.error("\n\u274C URL Validation Errors:");
|
|
405
|
+
errors.forEach(({ url: invalidUrl, error }) => {
|
|
406
|
+
console.error(` - ${invalidUrl}: ${error}`);
|
|
407
|
+
});
|
|
408
|
+
}
|
|
409
|
+
if (validUrls.length === 0) {
|
|
410
|
+
console.error("\n\u274C No valid URLs to process. Exiting.");
|
|
411
|
+
process.exit(1);
|
|
412
|
+
}
|
|
413
|
+
console.error(`
|
|
312
414
|
\u2705 Processing ${validUrls.length} valid URL(s)...`);
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
415
|
+
const pageFetcher = new PageFetcher(options.watch ? 0 : 1e4, 2, options.userAgent);
|
|
416
|
+
const execute = async () => {
|
|
417
|
+
const responses = await pageFetcher.fetchAll(validUrls);
|
|
418
|
+
const pageMetadatas = await buildPageMetadata(responses);
|
|
419
|
+
await printer.print(...pageMetadatas);
|
|
420
|
+
};
|
|
421
|
+
if (options.watch) {
|
|
422
|
+
process.stdin.resume();
|
|
423
|
+
process.on("SIGINT", () => process.exit(0));
|
|
424
|
+
let activeExecution = null;
|
|
425
|
+
process.stdin.on("end", () => {
|
|
426
|
+
activeExecution = null;
|
|
427
|
+
});
|
|
428
|
+
let winchTimer = null;
|
|
429
|
+
process.on("SIGWINCH", () => {
|
|
430
|
+
if (winchTimer !== null) clearTimeout(winchTimer);
|
|
431
|
+
winchTimer = setTimeout(() => {
|
|
432
|
+
winchTimer = null;
|
|
433
|
+
activeExecution = execute().catch((err) => {
|
|
434
|
+
console.error(
|
|
435
|
+
"\n\u274C An error occurred:",
|
|
436
|
+
err instanceof Error ? err.message : err
|
|
437
|
+
);
|
|
438
|
+
});
|
|
439
|
+
}, 150);
|
|
440
|
+
});
|
|
441
|
+
activeExecution = execute();
|
|
442
|
+
await activeExecution;
|
|
443
|
+
} else {
|
|
444
|
+
await execute();
|
|
445
|
+
}
|
|
446
|
+
} catch (error) {
|
|
447
|
+
console.error("\n\u274C An error occurred:", error instanceof Error ? error.message : error);
|
|
448
|
+
process.exit(1);
|
|
340
449
|
}
|
|
341
|
-
} catch (error) {
|
|
342
|
-
console.error("\n\u274C An error occurred:", error instanceof Error ? error.message : error);
|
|
343
|
-
process.exit(1);
|
|
344
450
|
}
|
|
345
|
-
|
|
451
|
+
);
|
|
346
452
|
program.command("file").description("extract resources from local file(s) via direct filesystem access").addArgument(fileArg).addOption(
|
|
347
453
|
new Option("--no-failsafe", `bypass the ${MAX_FILES_FAILSAFE}-file limit safety check`)
|
|
348
454
|
).action(async (paths, options) => {
|
|
349
455
|
try {
|
|
350
|
-
|
|
351
|
-
console.error(
|
|
352
|
-
`
|
|
353
|
-
\u274C ${paths.length} files specified exceeds the safety limit of ${MAX_FILES_FAILSAFE}.`
|
|
354
|
-
);
|
|
355
|
-
console.error(` Pass --no-failsafe to bypass this check and process all files.`);
|
|
356
|
-
process.exit(1);
|
|
357
|
-
}
|
|
358
|
-
if (!options.failsafe && paths.length > MAX_FILES_FAILSAFE) {
|
|
359
|
-
console.error(
|
|
360
|
-
`
|
|
361
|
-
\u26A0\uFE0F Failsafe bypassed: processing ${paths.length} files (limit is ${MAX_FILES_FAILSAFE}).`
|
|
362
|
-
);
|
|
363
|
-
}
|
|
364
|
-
console.error(`
|
|
365
|
-
\u2705 Processing ${paths.length} file(s)...`);
|
|
366
|
-
const fileFetcher = new FileFetcher();
|
|
367
|
-
const responses = await fileFetcher.fetchAll(paths);
|
|
368
|
-
const pageMetadatas = await buildPageMetadata(
|
|
369
|
-
responses.map(({ path, content, error }) => ({ path, content, error }))
|
|
370
|
-
);
|
|
371
|
-
await printer.print(...pageMetadatas);
|
|
456
|
+
await runFileCommand(paths, options);
|
|
372
457
|
} catch (error) {
|
|
373
458
|
console.error("\n\u274C An error occurred:", error instanceof Error ? error.message : error);
|
|
374
459
|
process.exit(1);
|
package/bin/main.js.map
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"version": 3,
|
|
3
3
|
"sources": ["../src/main.ts", "../src/extractors/AbstractExtractor.ts", "../src/extractors/PageExtractor.ts", "../src/resource.ts", "../src/extractors/ResourceExtractor.ts", "../src/page/PageFetcher.ts", "../src/page/FileFetcher.ts", "../src/printers/AbstractResourcePrinter.ts", "../src/printers/JSONStylePrinter.ts", "../src/security.ts"],
|
|
4
|
-
"sourcesContent": ["#!/usr/bin/env node\nimport { Command, createArgument, Option } from 'commander';\nimport { createRequire } from 'node:module';\nimport { resolve } from 'node:path';\nimport { fileURLToPath } from 'node:url';\nimport { realpathSync } from 'node:fs';\n\nimport { PageExtractor, ResourceExtractor } from './extractors/index.js';\nimport { FileFetcher, MAX_FILES_FAILSAFE, PageFetcher, type PageMetadata } from './page/index.js';\nimport { JSONStylePrinter } from './printers/index.js';\nimport { validateUrls } from './security.js';\n\nconst require = createRequire(import.meta.url);\nconst pkg = require('../package.json') as {\n description: string;\n name: string;\n version: string;\n};\n\nconst { description, name, version } = pkg;\n\nconst program = new Command();\n\nconst urlArg = createArgument('<url...>', 'remote https://URL to extract from');\nconst fileArg = createArgument('<paths...>', 'local file paths to extract from');\n\n// Shared extractor instances.\nconst pageExtractor = new PageExtractor();\nconst resourceExtractor = new ResourceExtractor(['a', 'meta', 'link', 'embed', 'script']);\nconst printer = new JSONStylePrinter();\n\nasync function buildPageMetadata(\n responses: Array<{\n url?: string;\n path?: string;\n content?: import('./page/index.js').DOMResult;\n error?: string;\n }>\n): Promise<PageMetadata[]> {\n const pageMetadatas: PageMetadata[] = [];\n\n for (const { content, url: responseUrl, path, error } of responses) {\n const resolvedUrl = responseUrl ?? path ?? '';\n const resources =\n error !== undefined || !content ? [] : await resourceExtractor.extract(content);\n const descriptor =\n error !== undefined || !content\n ? { url: resolvedUrl, error: error ?? 'Unknown error', resources }\n : await pageExtractor.extract(content);\n pageMetadatas.push({ ...descriptor, resources });\n }\n\n return pageMetadatas;\n}\n\nfunction isCliEntrypoint(): boolean {\n const invokedPath = process.argv[1];\n if (!invokedPath) {\n return false;\n }\n\n try {\n return realpathSync(fileURLToPath(import.meta.url)) === realpathSync(resolve(invokedPath));\n } catch {\n return false;\n }\n}\n\nexport async function runCli(argv: string[] = process.argv): Promise<void> {\n program.name(name).version(version, '-v, --version').description(description);\n\n // \u2500\u2500 fetch subcommand (default remote URL mode) \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n program\n .command('fetch', { isDefault: true })\n .description('fetch and extract resources from remote URL(s)')\n .addArgument(urlArg)\n .addOption(\n new Option(\n '--watch',\n 'keep running: SIGWINCH re-fetches after resize, Ctrl-D releases in-flight requests, Ctrl-C exits'\n )\n )\n .action(async (urls: string[], options: { watch: boolean }) => {\n try {\n const { validUrls, errors } = validateUrls(urls);\n\n if (errors.length > 0) {\n console.error('\\n\u274C URL Validation Errors:');\n errors.forEach(({ url: invalidUrl, error }) => {\n console.error(` - ${invalidUrl}: ${error}`);\n });\n }\n\n if (validUrls.length === 0) {\n console.error('\\n\u274C No valid URLs to process. Exiting.');\n process.exit(1);\n }\n\n console.error(`\\n\u2705 Processing ${validUrls.length} valid URL(s)...`);\n\n const pageFetcher = new PageFetcher(options.watch ? 0 : 10000, 2);\n\n const execute = async (): Promise<void> => {\n const responses = await pageFetcher.fetchAll(validUrls);\n const pageMetadatas = await buildPageMetadata(responses);\n await printer.print(...pageMetadatas);\n };\n\n if (options.watch) {\n process.stdin.resume();\n process.on('SIGINT', () => process.exit(0));\n\n let activeExecution: Promise<void> | null = null;\n process.stdin.on('end', () => {\n activeExecution = null;\n });\n\n let winchTimer: ReturnType<typeof setTimeout> | null = null;\n process.on('SIGWINCH', () => {\n if (winchTimer !== null) clearTimeout(winchTimer);\n winchTimer = setTimeout(() => {\n winchTimer = null;\n activeExecution = execute().catch((err: unknown) => {\n console.error('\\n\u274C An error occurred:', err instanceof Error ? err.message : err);\n });\n }, 150);\n });\n\n activeExecution = execute();\n await activeExecution;\n } else {\n await execute();\n }\n } catch (error) {\n console.error('\\n\u274C An error occurred:', error instanceof Error ? error.message : error);\n process.exit(1);\n }\n });\n\n // \u2500\u2500 file subcommand (local filesystem access) \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n program\n .command('file')\n .description('extract resources from local file(s) via direct filesystem access')\n .addArgument(fileArg)\n .addOption(\n new Option('--no-failsafe', `bypass the ${MAX_FILES_FAILSAFE}-file limit safety check`)\n )\n .action(async (paths: string[], options: { failsafe: boolean }) => {\n try {\n if (options.failsafe && paths.length > MAX_FILES_FAILSAFE) {\n console.error(\n `\\n\u274C ${paths.length} files specified exceeds the safety limit of ${MAX_FILES_FAILSAFE}.`\n );\n console.error(` Pass --no-failsafe to bypass this check and process all files.`);\n process.exit(1);\n }\n\n if (!options.failsafe && paths.length > MAX_FILES_FAILSAFE) {\n console.error(\n `\\n\u26A0\uFE0F Failsafe bypassed: processing ${paths.length} files (limit is ${MAX_FILES_FAILSAFE}).`\n );\n }\n\n console.error(`\\n\u2705 Processing ${paths.length} file(s)...`);\n\n const fileFetcher = new FileFetcher();\n const responses = await fileFetcher.fetchAll(paths);\n const pageMetadatas = await buildPageMetadata(\n responses.map(({ path, content, error }) => ({ path, content, error }))\n );\n\n await printer.print(...pageMetadatas);\n } catch (error) {\n console.error('\\n\u274C An error occurred:', error instanceof Error ? error.message : error);\n process.exit(1);\n }\n });\n\n await program.parseAsync(argv);\n}\n\nif (isCliEntrypoint()) {\n runCli().catch((error: unknown) => {\n console.error('\\n\u274C An error occurred:', error instanceof Error ? error.message : error);\n process.exit(1);\n });\n}\n", "export abstract class AbstractExtractor<V, R> {\n constructor(readonly name: string) {}\n abstract extract(value: V): Promise<R>;\n}\n", "import type { Page } from '../page/index.js';\nimport type { DOMResult } from '../page/index.js';\nimport { AbstractExtractor } from './AbstractExtractor.js';\n\nexport class PageExtractor extends AbstractExtractor<DOMResult, Page> {\n constructor() {\n super('page-extractor');\n }\n\n async extract(value: DOMResult): Promise<Page> {\n const {\n window: { document },\n url,\n } = value;\n return { title: document.title, url };\n }\n}\n", "/**\n * @license MIT\n * We are interested in visualising a page as a collection of tags.\n *\n * We wish to work with tags that can be compactly previewed on a webpage.\n * Here we must declare all of the element types that can be used to represent\n * a resource that can be hyperlinked off a webpage.\n */\ntype Tags = HTMLElementTagNameMap;\n\nexport const RESOURCE_DISPLAYABLE_KEYS = [\n 'id',\n 'innerText',\n 'textContent',\n 'class',\n 'ariaLabel',\n 'ariaDescription',\n 'alt',\n] as const;\n\nexport type DisplayableKey = (typeof RESOURCE_DISPLAYABLE_KEYS)[number];\n\nexport const RESOURCE_LINK_KEYS = ['href', 'data-src', 'target', 'action', 'src', 'url'] as const;\n\nexport type LinkKey = (typeof RESOURCE_LINK_KEYS)[number];\n\nexport type AttributeKey = DisplayableKey | LinkKey;\n\nexport type ResourceKey = { key: AttributeKey; value: string };\nexport type ResourceLink = { key: LinkKey; value: string };\n\nexport type ExternalResource = {\n text: ResourceKey;\n link: ResourceLink;\n};\n\nexport type Tag = keyof Tags;\n\nexport type Resource = HTMLElement & {\n [K in AttributeKey]?: string | null;\n};\n\nexport type ResourceByName<T extends keyof Tags> = Tags[T];\n\n// --- adapters ---\n\nconst readAttr = (element: Resource, key: AttributeKey): string | undefined => {\n const v = element.getAttribute(key);\n return v != null && v.trim() !== '' ? v : undefined;\n};\n\nexport function findResourceText(element: Resource): ResourceKey | undefined {\n for (const key of RESOURCE_DISPLAYABLE_KEYS) {\n const value = readAttr(element, key);\n if (value !== undefined) return { key, value };\n }\n return undefined;\n}\n\nexport function findResourceLink(element: Resource): ResourceLink | undefined {\n for (const key of RESOURCE_LINK_KEYS) {\n const value = readAttr(element, key);\n if (value !== undefined) return { key, value };\n }\n return undefined;\n}\n\nexport const isResourceKey = (key: string): key is AttributeKey =>\n (RESOURCE_DISPLAYABLE_KEYS as readonly string[]).includes(key) ||\n (RESOURCE_LINK_KEYS as readonly string[]).includes(key);\n", "import type { DOMResult } from '../page/index.js';\nimport {\n findResourceLink,\n findResourceText,\n type ExternalResource,\n type Resource,\n type Tag,\n} from '../resource.js';\nimport { AbstractExtractor } from './AbstractExtractor.js';\n\nexport class ResourceExtractor extends AbstractExtractor<DOMResult, ExternalResource[]> {\n constructor(private readonly tags: Tag[]) {\n super('page-extractor');\n }\n async extract(value: DOMResult): Promise<ExternalResource[]> {\n const { document } = value.window;\n return this.tags.flatMap((tag) =>\n Array.from(document.querySelectorAll<Resource>(tag)).flatMap((element) => {\n const link = findResourceLink(element);\n if (!link) return [];\n const text = findResourceText(element) ?? { key: 'src' as const, value: link.value };\n return [{ text, link }];\n })\n );\n }\n}\n", "import { parseHTML } from 'linkedom';\n\ntype ParseHTMLResult = {\n document: Document;\n};\n\nexport interface DOMResult {\n window: { document: Document };\n url: string;\n}\n\ninterface PageResponse {\n url: string;\n content?: DOMResult;\n error?: string;\n}\n\nexport class PageFetcher {\n private readonly timeout: number;\n private readonly maxRetries: number;\n\n constructor(timeout = 10000, maxRetries = 2) {\n this.timeout = timeout;\n this.maxRetries = maxRetries;\n }\n\n private buildDOMResult(html: string, url: string): DOMResult {\n const { document } = parseHTML(html) as ParseHTMLResult;\n return { window: { document }, url };\n }\n\n private decodeHtml(buffer: ArrayBuffer, charset: string): string {\n try {\n return new TextDecoder(charset).decode(new Uint8Array(buffer));\n } catch {\n return new TextDecoder('utf-8').decode(new Uint8Array(buffer));\n }\n }\n\n private async fetchPage(url: string, retryCount = 0): Promise<PageResponse> {\n const controller = new AbortController();\n let timeoutId: ReturnType<typeof setTimeout> | null = null;\n\n try {\n if (this.timeout > 0) {\n timeoutId = setTimeout(() => {\n controller.abort(new Error('Request timeout'));\n }, this.timeout);\n }\n\n const content = await fetch(url, { signal: controller.signal }).then(async (response) => {\n const buffer = await response.arrayBuffer();\n const contentType = response.headers.get('content-type') ?? '';\n const charsetMatch = /charset=([^\\s;]+)/i.exec(contentType);\n const html = this.decodeHtml(buffer, charsetMatch?.[1] ?? 'utf-8');\n return this.buildDOMResult(html, url);\n });\n\n return { url, content };\n } catch (error) {\n const abortTimeout = error instanceof Error && error.name === 'AbortError';\n const message = abortTimeout\n ? 'Request timeout'\n : error instanceof Error\n ? error.message\n : 'Unknown error';\n\n // Retry logic for transient errors\n if (retryCount < this.maxRetries && this.isRetryableError(message)) {\n process.stderr.write(`Retrying ${url} (attempt ${retryCount + 1}/${this.maxRetries})...\\n`);\n await this.delay(1000 * (retryCount + 1)); // Exponential backoff\n return this.fetchPage(url, retryCount + 1);\n }\n\n return { url, error: `Failed to fetch: ${message}` };\n } finally {\n if (timeoutId !== null) {\n clearTimeout(timeoutId);\n }\n }\n }\n\n private isRetryableError(message: string): boolean {\n const retryablePatterns = [/timeout/i, /ECONNRESET/i, /ETIMEDOUT/i, /ENOTFOUND/i, /network/i];\n return retryablePatterns.some((pattern) => pattern.test(message));\n }\n\n private delay(ms: number): Promise<void> {\n return new Promise((resolve) => setTimeout(resolve, ms));\n }\n\n async fetchAll(urls: string[]): Promise<PageResponse[]> {\n const responses = await Promise.all(urls.map((url) => this.fetchPage(url)));\n return responses.filter((response) => response.content !== undefined || response.error);\n }\n}\n", "import { readFile } from 'node:fs/promises';\nimport { parseHTML } from 'linkedom';\n\nimport type { DOMResult } from './PageFetcher.js';\n\nexport const MAX_FILES_FAILSAFE = 254;\n\ntype ParseHTMLResult = {\n document: Document;\n};\n\nexport interface FileResponse {\n path: string;\n content?: DOMResult;\n error?: string;\n}\n\nexport class FileFetcher {\n private buildDOMResult(html: string, filePath: string): DOMResult {\n const { document } = parseHTML(html) as ParseHTMLResult;\n return { window: { document }, url: `file://${filePath}` };\n }\n\n async fetchFile(filePath: string): Promise<FileResponse> {\n try {\n // filePath is supplied directly by the CLI user, not derived from network input.\n // eslint-disable-next-line security/detect-non-literal-fs-filename\n const html = await readFile(filePath, 'utf-8');\n return { path: filePath, content: this.buildDOMResult(html, filePath) };\n } catch (error) {\n return {\n path: filePath,\n error: error instanceof Error ? error.message : 'Unknown error',\n };\n }\n }\n\n async fetchAll(filePaths: string[]): Promise<FileResponse[]> {\n return Promise.all(filePaths.map((p) => this.fetchFile(p)));\n }\n}\n", "import type { PageMetadata } from '../page/index.js';\n\nexport abstract class AbstractResourcePrinter {\n constructor() {}\n abstract print(...pages: PageMetadata[]): void | Promise<void>;\n}\n", "import type { PageMetadata } from '../page/index.js';\nimport { AbstractResourcePrinter } from './AbstractResourcePrinter.js';\n\nexport class JSONStylePrinter extends AbstractResourcePrinter {\n print(...pages: PageMetadata[]): void | Promise<void> {\n const json = JSON.stringify(pages);\n process.stdout.write(json + '\\n');\n }\n}\n", "/**\n * Security utilities for URL validation and sanitization\n */\n\nconst ALLOWED_PROTOCOLS = ['http:', 'https:'];\nconst MAX_URL_LENGTH = 2048;\nconst SUSPICIOUS_PATTERNS = [\n /javascript:/i,\n /data:/i,\n /vbscript:/i,\n /<script/i,\n /on\\w+=/i, // Event handlers like onclick=\n];\n\nexport interface ValidationResult {\n isValid: boolean;\n error?: string;\n sanitizedUrl?: string;\n}\n\n/**\n * Validates a URL for security concerns\n * @param url - The URL to validate\n * @returns ValidationResult object with validation status\n */\nexport function validateUrl(url: string): ValidationResult {\n // Check if URL is empty or whitespace\n if (!url || !url.trim()) {\n return {\n isValid: false,\n error: 'URL cannot be empty',\n };\n }\n\n const trimmedUrl = url.trim();\n\n // Check URL length to prevent DoS\n if (trimmedUrl.length > MAX_URL_LENGTH) {\n return {\n isValid: false,\n error: `URL exceeds maximum length of ${MAX_URL_LENGTH} characters`,\n };\n }\n\n // Check for suspicious patterns\n for (const pattern of SUSPICIOUS_PATTERNS) {\n if (pattern.test(trimmedUrl)) {\n return {\n isValid: false,\n error: 'URL contains suspicious patterns',\n };\n }\n }\n\n // Parse the URL\n let parsedUrl: URL;\n try {\n parsedUrl = new URL(trimmedUrl);\n } catch {\n return {\n isValid: false,\n error: 'Invalid URL format',\n };\n }\n\n // Check protocol\n if (!ALLOWED_PROTOCOLS.includes(parsedUrl.protocol)) {\n return {\n isValid: false,\n error: `Protocol ${parsedUrl.protocol} is not allowed. Allowed protocols: ${ALLOWED_PROTOCOLS.join(', ')}`,\n };\n }\n\n // Check for localhost/internal IPs in production (security consideration)\n const hostname = parsedUrl.hostname.toLowerCase();\n const isLocalhost =\n hostname === 'localhost' ||\n hostname === '127.0.0.1' ||\n hostname === '::1' ||\n hostname.startsWith('192.168.') ||\n hostname.startsWith('10.') ||\n /^172\\.(1[6-9]|2\\d|3[01])\\./.test(hostname);\n\n if (isLocalhost) {\n // Allow but warn about localhost URLs\n console.warn(`Warning: Accessing local network resource: ${trimmedUrl}`);\n }\n\n return {\n isValid: true,\n sanitizedUrl: parsedUrl.toString(),\n };\n}\n\n/**\n * Validates an array of URLs\n * @param urls - Array of URLs to validate\n * @returns Object with valid URLs and errors\n */\nexport function validateUrls(urls: string[]): {\n validUrls: string[];\n errors: Array<{ url: string; error: string }>;\n} {\n const validUrls: string[] = [];\n const errors: Array<{ url: string; error: string }> = [];\n\n for (const url of urls) {\n const result = validateUrl(url);\n if (result.isValid && result.sanitizedUrl) {\n validUrls.push(result.sanitizedUrl);\n } else {\n errors.push({\n url,\n error: result.error || 'Unknown validation error',\n });\n }\n }\n\n return { validUrls, errors };\n}\n\n/**\n * Rate limiter to prevent abuse\n */\nexport class RateLimiter {\n private requests: number[] = [];\n private readonly maxRequests: number;\n private readonly windowMs: number;\n\n constructor(maxRequests = 10, windowMs = 60000) {\n this.maxRequests = maxRequests;\n this.windowMs = windowMs;\n }\n\n /**\n * Check if a request is allowed under rate limiting\n * @returns true if request is allowed, false otherwise\n */\n public isAllowed(): boolean {\n const now = Date.now();\n\n // Remove old requests outside the time window\n this.requests = this.requests.filter((time) => now - time < this.windowMs);\n\n if (this.requests.length >= this.maxRequests) {\n return false;\n }\n\n this.requests.push(now);\n return true;\n }\n\n /**\n * Get remaining requests in current window\n */\n public getRemainingRequests(): number {\n const now = Date.now();\n this.requests = this.requests.filter((time) => now - time < this.windowMs);\n return Math.max(0, this.maxRequests - this.requests.length);\n }\n}\n\n/**\n * Sanitizes HTML content to prevent XSS attacks\n * @param text - Text to sanitize\n * @returns Sanitized text\n */\nexport function sanitizeText(text: string): string {\n if (!text) return '';\n\n return text\n .replace(/</g, '<')\n .replace(/>/g, '>')\n .replace(/\"/g, '"')\n .replace(/'/g, ''')\n .replace(/\\//g, '/');\n}\n"],
|
|
5
|
-
"mappings": ";;;AACA,SAAS,SAAS,gBAAgB,cAAc;AAChD,SAAS,qBAAqB;AAC9B,SAAS,eAAe;AACxB,SAAS,qBAAqB;AAC9B,SAAS,oBAAoB;;;ACLtB,IAAe,oBAAf,MAAuC;AAAA,EAC5C,YAAqBA,OAAc;AAAd,gBAAAA;AAAA,EAAe;AAEtC;;;ACCO,IAAM,gBAAN,cAA4B,kBAAmC;AAAA,EACpE,cAAc;AACZ,UAAM,gBAAgB;AAAA,EACxB;AAAA,EAEA,MAAM,QAAQ,OAAiC;AAC7C,UAAM;AAAA,MACJ,QAAQ,EAAE,SAAS;AAAA,MACnB;AAAA,IACF,IAAI;AACJ,WAAO,EAAE,OAAO,SAAS,OAAO,IAAI;AAAA,EACtC;AACF;;;ACNO,IAAM,4BAA4B;AAAA,EACvC;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AACF;AAIO,IAAM,qBAAqB,CAAC,QAAQ,YAAY,UAAU,UAAU,OAAO,KAAK;AAwBvF,IAAM,WAAW,CAAC,SAAmB,QAA0C;AAC7E,QAAM,IAAI,QAAQ,aAAa,GAAG;AAClC,SAAO,KAAK,QAAQ,EAAE,KAAK,MAAM,KAAK,IAAI;AAC5C;AAEO,SAAS,iBAAiB,SAA4C;AAC3E,aAAW,OAAO,2BAA2B;AAC3C,UAAM,QAAQ,SAAS,SAAS,GAAG;AACnC,QAAI,UAAU,OAAW,QAAO,EAAE,KAAK,MAAM;AAAA,EAC/C;AACA,SAAO;AACT;AAEO,SAAS,iBAAiB,SAA6C;AAC5E,aAAW,OAAO,oBAAoB;AACpC,UAAM,QAAQ,SAAS,SAAS,GAAG;AACnC,QAAI,UAAU,OAAW,QAAO,EAAE,KAAK,MAAM;AAAA,EAC/C;AACA,SAAO;AACT;;;ACvDO,IAAM,oBAAN,cAAgC,kBAAiD;AAAA,EACtF,YAA6B,MAAa;AACxC,UAAM,gBAAgB;AADK;AAAA,EAE7B;AAAA,EACA,MAAM,QAAQ,OAA+C;AAC3D,UAAM,EAAE,SAAS,IAAI,MAAM;AAC3B,WAAO,KAAK,KAAK;AAAA,MAAQ,CAAC,QACxB,MAAM,KAAK,SAAS,iBAA2B,GAAG,CAAC,EAAE,QAAQ,CAAC,YAAY;AACxE,cAAM,OAAO,iBAAiB,OAAO;AACrC,YAAI,CAAC,KAAM,QAAO,CAAC;AACnB,cAAM,OAAO,iBAAiB,OAAO,KAAK,EAAE,KAAK,OAAgB,OAAO,KAAK,MAAM;AACnF,eAAO,CAAC,EAAE,MAAM,KAAK,CAAC;AAAA,MACxB,CAAC;AAAA,IACH;AAAA,EACF;AACF;;;ACzBA,SAAS,iBAAiB;AAiBnB,IAAM,cAAN,MAAkB;AAAA,EACN;AAAA,EACA;AAAA,EAEjB,YAAY,UAAU,KAAO,aAAa,GAAG;AAC3C,SAAK,UAAU;AACf,SAAK,aAAa;AAAA,EACpB;AAAA,EAEQ,eAAe,MAAc,KAAwB;AAC3D,UAAM,EAAE,SAAS,IAAI,UAAU,IAAI;AACnC,WAAO,EAAE,QAAQ,EAAE,SAAS,GAAG,IAAI;AAAA,EACrC;AAAA,EAEQ,WAAW,QAAqB,SAAyB;AAC/D,QAAI;AACF,aAAO,IAAI,YAAY,OAAO,EAAE,OAAO,IAAI,WAAW,MAAM,CAAC;AAAA,IAC/D,QAAQ;AACN,aAAO,IAAI,YAAY,OAAO,EAAE,OAAO,IAAI,WAAW,MAAM,CAAC;AAAA,IAC/D;AAAA,EACF;AAAA,EAEA,MAAc,UAAU,KAAa,aAAa,GAA0B;AAC1E,UAAM,aAAa,IAAI,gBAAgB;AACvC,QAAI,YAAkD;AAEtD,QAAI;AACF,UAAI,KAAK,UAAU,GAAG;AACpB,oBAAY,WAAW,MAAM;AAC3B,qBAAW,MAAM,IAAI,MAAM,iBAAiB,CAAC;AAAA,QAC/C,GAAG,KAAK,OAAO;AAAA,MACjB;AAEA,YAAM,UAAU,MAAM,MAAM,KAAK,EAAE,QAAQ,WAAW,OAAO,CAAC,EAAE,KAAK,OAAO,aAAa;AACvF,cAAM,SAAS,MAAM,SAAS,YAAY;AAC1C,cAAM,cAAc,SAAS,QAAQ,IAAI,cAAc,KAAK;AAC5D,cAAM,eAAe,qBAAqB,KAAK,WAAW;AAC1D,cAAM,OAAO,KAAK,WAAW,QAAQ,eAAe,CAAC,KAAK,OAAO;AACjE,eAAO,KAAK,eAAe,MAAM,GAAG;AAAA,MACtC,CAAC;AAED,aAAO,EAAE,KAAK,QAAQ;AAAA,IACxB,SAAS,OAAO;AACd,YAAM,eAAe,iBAAiB,SAAS,MAAM,SAAS;AAC9D,YAAM,UAAU,eACZ,oBACA,iBAAiB,QACf,MAAM,UACN;AAGN,UAAI,aAAa,KAAK,cAAc,KAAK,iBAAiB,OAAO,GAAG;AAClE,gBAAQ,OAAO,MAAM,YAAY,GAAG,aAAa,aAAa,CAAC,IAAI,KAAK,UAAU;AAAA,CAAQ;AAC1F,cAAM,KAAK,MAAM,OAAQ,aAAa,EAAE;AACxC,eAAO,KAAK,UAAU,KAAK,aAAa,CAAC;AAAA,MAC3C;AAEA,aAAO,EAAE,KAAK,OAAO,oBAAoB,OAAO,GAAG;AAAA,IACrD,UAAE;AACA,UAAI,cAAc,MAAM;AACtB,qBAAa,SAAS;AAAA,MACxB;AAAA,IACF;AAAA,EACF;AAAA,EAEQ,iBAAiB,SAA0B;AACjD,UAAM,oBAAoB,CAAC,YAAY,eAAe,cAAc,cAAc,UAAU;AAC5F,WAAO,kBAAkB,KAAK,CAAC,YAAY,QAAQ,KAAK,OAAO,CAAC;AAAA,EAClE;AAAA,EAEQ,MAAM,IAA2B;AACvC,WAAO,IAAI,QAAQ,CAACC,aAAY,WAAWA,UAAS,EAAE,CAAC;AAAA,EACzD;AAAA,EAEA,MAAM,SAAS,MAAyC;AACtD,UAAM,YAAY,MAAM,QAAQ,IAAI,KAAK,IAAI,CAAC,QAAQ,KAAK,UAAU,GAAG,CAAC,CAAC;AAC1E,WAAO,UAAU,OAAO,CAAC,aAAa,SAAS,YAAY,UAAa,SAAS,KAAK;AAAA,EACxF;AACF;;;AC/FA,SAAS,gBAAgB;AACzB,SAAS,aAAAC,kBAAiB;AAInB,IAAM,qBAAqB;AAY3B,IAAM,cAAN,MAAkB;AAAA,EACf,eAAe,MAAc,UAA6B;AAChE,UAAM,EAAE,SAAS,IAAIA,WAAU,IAAI;AACnC,WAAO,EAAE,QAAQ,EAAE,SAAS,GAAG,KAAK,UAAU,QAAQ,GAAG;AAAA,EAC3D;AAAA,EAEA,MAAM,UAAU,UAAyC;AACvD,QAAI;AAGF,YAAM,OAAO,MAAM,SAAS,UAAU,OAAO;AAC7C,aAAO,EAAE,MAAM,UAAU,SAAS,KAAK,eAAe,MAAM,QAAQ,EAAE;AAAA,IACxE,SAAS,OAAO;AACd,aAAO;AAAA,QACL,MAAM;AAAA,QACN,OAAO,iBAAiB,QAAQ,MAAM,UAAU;AAAA,MAClD;AAAA,IACF;AAAA,EACF;AAAA,EAEA,MAAM,SAAS,WAA8C;AAC3D,WAAO,QAAQ,IAAI,UAAU,IAAI,CAAC,MAAM,KAAK,UAAU,CAAC,CAAC,CAAC;AAAA,EAC5D;AACF;;;ACtCO,IAAe,0BAAf,MAAuC;AAAA,EAC5C,cAAc;AAAA,EAAC;AAEjB;;;ACFO,IAAM,mBAAN,cAA+B,wBAAwB;AAAA,EAC5D,SAAS,OAA6C;AACpD,UAAM,OAAO,KAAK,UAAU,KAAK;AACjC,YAAQ,OAAO,MAAM,OAAO,IAAI;AAAA,EAClC;AACF;;;ACJA,IAAM,oBAAoB,CAAC,SAAS,QAAQ;AAC5C,IAAM,iBAAiB;AACvB,IAAM,sBAAsB;AAAA,EAC1B;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA;AACF;AAaO,SAAS,YAAY,KAA+B;AAEzD,MAAI,CAAC,OAAO,CAAC,IAAI,KAAK,GAAG;AACvB,WAAO;AAAA,MACL,SAAS;AAAA,MACT,OAAO;AAAA,IACT;AAAA,EACF;AAEA,QAAM,aAAa,IAAI,KAAK;AAG5B,MAAI,WAAW,SAAS,gBAAgB;AACtC,WAAO;AAAA,MACL,SAAS;AAAA,MACT,OAAO,iCAAiC,cAAc;AAAA,IACxD;AAAA,EACF;AAGA,aAAW,WAAW,qBAAqB;AACzC,QAAI,QAAQ,KAAK,UAAU,GAAG;AAC5B,aAAO;AAAA,QACL,SAAS;AAAA,QACT,OAAO;AAAA,MACT;AAAA,IACF;AAAA,EACF;AAGA,MAAI;AACJ,MAAI;AACF,gBAAY,IAAI,IAAI,UAAU;AAAA,EAChC,QAAQ;AACN,WAAO;AAAA,MACL,SAAS;AAAA,MACT,OAAO;AAAA,IACT;AAAA,EACF;AAGA,MAAI,CAAC,kBAAkB,SAAS,UAAU,QAAQ,GAAG;AACnD,WAAO;AAAA,MACL,SAAS;AAAA,MACT,OAAO,YAAY,UAAU,QAAQ,uCAAuC,kBAAkB,KAAK,IAAI,CAAC;AAAA,IAC1G;AAAA,EACF;AAGA,QAAM,WAAW,UAAU,SAAS,YAAY;AAChD,QAAM,cACJ,aAAa,eACb,aAAa,eACb,aAAa,SACb,SAAS,WAAW,UAAU,KAC9B,SAAS,WAAW,KAAK,KACzB,6BAA6B,KAAK,QAAQ;AAE5C,MAAI,aAAa;AAEf,YAAQ,KAAK,8CAA8C,UAAU,EAAE;AAAA,EACzE;AAEA,SAAO;AAAA,IACL,SAAS;AAAA,IACT,cAAc,UAAU,SAAS;AAAA,EACnC;AACF;AAOO,SAAS,aAAa,MAG3B;AACA,QAAM,YAAsB,CAAC;AAC7B,QAAM,SAAgD,CAAC;AAEvD,aAAW,OAAO,MAAM;AACtB,UAAM,SAAS,YAAY,GAAG;AAC9B,QAAI,OAAO,WAAW,OAAO,cAAc;AACzC,gBAAU,KAAK,OAAO,YAAY;AAAA,IACpC,OAAO;AACL,aAAO,KAAK;AAAA,QACV;AAAA,QACA,OAAO,OAAO,SAAS;AAAA,MACzB,CAAC;AAAA,IACH;AAAA,EACF;AAEA,SAAO,EAAE,WAAW,OAAO;AAC7B;;;AT3GA,IAAMC,WAAU,cAAc,YAAY,GAAG;AAC7C,IAAM,MAAMA,SAAQ,iBAAiB;AAMrC,IAAM,EAAE,aAAa,MAAM,QAAQ,IAAI;AAEvC,IAAM,UAAU,IAAI,QAAQ;AAE5B,IAAM,SAAS,eAAe,YAAY,oCAAoC;AAC9E,IAAM,UAAU,eAAe,cAAc,kCAAkC;AAG/E,IAAM,gBAAgB,IAAI,cAAc;AACxC,IAAM,oBAAoB,IAAI,kBAAkB,CAAC,KAAK,QAAQ,QAAQ,SAAS,QAAQ,CAAC;AACxF,IAAM,UAAU,IAAI,iBAAiB;AAErC,eAAe,kBACb,WAMyB;AACzB,QAAM,gBAAgC,CAAC;AAEvC,aAAW,EAAE,SAAS,KAAK,aAAa,MAAM,MAAM,KAAK,WAAW;AAClE,UAAM,cAAc,eAAe,QAAQ;AAC3C,UAAM,YACJ,UAAU,UAAa,CAAC,UAAU,CAAC,IAAI,MAAM,kBAAkB,QAAQ,OAAO;AAChF,UAAM,aACJ,UAAU,UAAa,CAAC,UACpB,EAAE,KAAK,aAAa,OAAO,SAAS,iBAAiB,UAAU,IAC/D,MAAM,cAAc,QAAQ,OAAO;AACzC,kBAAc,KAAK,EAAE,GAAG,YAAY,UAAU,CAAC;AAAA,EACjD;AAEA,SAAO;AACT;AAEA,SAAS,kBAA2B;AAClC,QAAM,cAAc,QAAQ,KAAK,CAAC;AAClC,MAAI,CAAC,aAAa;AAChB,WAAO;AAAA,EACT;AAEA,MAAI;AACF,WAAO,aAAa,cAAc,YAAY,GAAG,CAAC,MAAM,aAAa,QAAQ,WAAW,CAAC;AAAA,EAC3F,QAAQ;AACN,WAAO;AAAA,EACT;AACF;AAEA,eAAsB,OAAO,OAAiB,QAAQ,MAAqB;AACzE,UAAQ,KAAK,IAAI,EAAE,QAAQ,SAAS,eAAe,EAAE,YAAY,WAAW;AAG5E,UACG,QAAQ,SAAS,EAAE,WAAW,KAAK,CAAC,EACpC,YAAY,gDAAgD,EAC5D,YAAY,MAAM,EAClB;AAAA,IACC,IAAI;AAAA,MACF;AAAA,MACA;AAAA,IACF;AAAA,EACF,EACC,OAAO,OAAO,MAAgB,YAAgC;AAC7D,QAAI;AACF,YAAM,EAAE,WAAW,OAAO,IAAI,aAAa,IAAI;AAE/C,UAAI,OAAO,SAAS,GAAG;AACrB,gBAAQ,MAAM,iCAA4B;AAC1C,eAAO,QAAQ,CAAC,EAAE,KAAK,YAAY,MAAM,MAAM;AAC7C,kBAAQ,MAAM,OAAO,UAAU,KAAK,KAAK,EAAE;AAAA,QAC7C,CAAC;AAAA,MACH;AAEA,UAAI,UAAU,WAAW,GAAG;AAC1B,gBAAQ,MAAM,6CAAwC;AACtD,gBAAQ,KAAK,CAAC;AAAA,MAChB;AAEA,cAAQ,MAAM;AAAA,oBAAkB,UAAU,MAAM,kBAAkB;AAElE,YAAM,cAAc,IAAI,YAAY,QAAQ,QAAQ,IAAI,KAAO,CAAC;AAEhE,YAAM,UAAU,YAA2B;AACzC,cAAM,YAAY,MAAM,YAAY,SAAS,SAAS;AACtD,cAAM,gBAAgB,MAAM,kBAAkB,SAAS;AACvD,cAAM,QAAQ,MAAM,GAAG,aAAa;AAAA,MACtC;AAEA,UAAI,QAAQ,OAAO;AACjB,gBAAQ,MAAM,OAAO;AACrB,gBAAQ,GAAG,UAAU,MAAM,QAAQ,KAAK,CAAC,CAAC;AAE1C,YAAI,kBAAwC;AAC5C,gBAAQ,MAAM,GAAG,OAAO,MAAM;AAC5B,4BAAkB;AAAA,QACpB,CAAC;AAED,YAAI,aAAmD;AACvD,gBAAQ,GAAG,YAAY,MAAM;AAC3B,cAAI,eAAe,KAAM,cAAa,UAAU;AAChD,uBAAa,WAAW,MAAM;AAC5B,yBAAa;AACb,8BAAkB,QAAQ,EAAE,MAAM,CAAC,QAAiB;AAClD,sBAAQ,MAAM,+BAA0B,eAAe,QAAQ,IAAI,UAAU,GAAG;AAAA,YAClF,CAAC;AAAA,UACH,GAAG,GAAG;AAAA,QACR,CAAC;AAED,0BAAkB,QAAQ;AAC1B,cAAM;AAAA,MACR,OAAO;AACL,cAAM,QAAQ;AAAA,MAChB;AAAA,IACF,SAAS,OAAO;AACd,cAAQ,MAAM,+BAA0B,iBAAiB,QAAQ,MAAM,UAAU,KAAK;AACtF,cAAQ,KAAK,CAAC;AAAA,IAChB;AAAA,EACF,CAAC;AAGH,UACG,QAAQ,MAAM,EACd,YAAY,mEAAmE,EAC/E,YAAY,OAAO,EACnB;AAAA,IACC,IAAI,OAAO,iBAAiB,cAAc,kBAAkB,0BAA0B;AAAA,EACxF,EACC,OAAO,OAAO,OAAiB,YAAmC;AACjE,QAAI;AACF,UAAI,QAAQ,YAAY,MAAM,SAAS,oBAAoB;AACzD,gBAAQ;AAAA,UACN;AAAA,SAAO,MAAM,MAAM,gDAAgD,kBAAkB;AAAA,QACvF;AACA,gBAAQ,MAAM,mEAAmE;AACjF,gBAAQ,KAAK,CAAC;AAAA,MAChB;AAEA,UAAI,CAAC,QAAQ,YAAY,MAAM,SAAS,oBAAoB;AAC1D,gBAAQ;AAAA,UACN;AAAA,8CAAuC,MAAM,MAAM,oBAAoB,kBAAkB;AAAA,QAC3F;AAAA,MACF;AAEA,cAAQ,MAAM;AAAA,oBAAkB,MAAM,MAAM,aAAa;AAEzD,YAAM,cAAc,IAAI,YAAY;AACpC,YAAM,YAAY,MAAM,YAAY,SAAS,KAAK;AAClD,YAAM,gBAAgB,MAAM;AAAA,QAC1B,UAAU,IAAI,CAAC,EAAE,MAAM,SAAS,MAAM,OAAO,EAAE,MAAM,SAAS,MAAM,EAAE;AAAA,MACxE;AAEA,YAAM,QAAQ,MAAM,GAAG,aAAa;AAAA,IACtC,SAAS,OAAO;AACd,cAAQ,MAAM,+BAA0B,iBAAiB,QAAQ,MAAM,UAAU,KAAK;AACtF,cAAQ,KAAK,CAAC;AAAA,IAChB;AAAA,EACF,CAAC;AAEH,QAAM,QAAQ,WAAW,IAAI;AAC/B;AAEA,IAAI,gBAAgB,GAAG;AACrB,SAAO,EAAE,MAAM,CAAC,UAAmB;AACjC,YAAQ,MAAM,+BAA0B,iBAAiB,QAAQ,MAAM,UAAU,KAAK;AACtF,YAAQ,KAAK,CAAC;AAAA,EAChB,CAAC;AACH;",
|
|
4
|
+
"sourcesContent": ["#!/usr/bin/env node\nimport { Command, createArgument, Option } from 'commander';\nimport { createRequire } from 'node:module';\nimport { resolve } from 'node:path';\nimport { fileURLToPath } from 'node:url';\nimport { realpathSync } from 'node:fs';\n\nimport { PageExtractor, ResourceExtractor } from './extractors/index.js';\nimport { FileFetcher, MAX_FILES_FAILSAFE, PageFetcher, type PageMetadata } from './page/index.js';\nimport { JSONStylePrinter } from './printers/index.js';\nimport { validateUrls } from './security.js';\n\nconst require = createRequire(import.meta.url);\nconst pkg = require('../package.json') as {\n description: string;\n name: string;\n version: string;\n};\n\nconst { description, name, version } = pkg;\n\nconst program = new Command();\n\nconst urlArg = createArgument('<url...>', 'remote https://URL to extract from');\nconst fileArg = createArgument('<paths...>', 'local file paths to extract from');\n\n// Shared extractor instances.\nconst pageExtractor = new PageExtractor();\nconst resourceExtractor = new ResourceExtractor(['a', 'meta', 'link', 'embed', 'script']);\nconst printer = new JSONStylePrinter();\n\nasync function buildPageMetadata(\n responses: Array<{\n url?: string;\n path?: string;\n content?: import('./page/index.js').DOMResult;\n error?: string;\n }>\n): Promise<PageMetadata[]> {\n const pageMetadatas: PageMetadata[] = [];\n\n for (const { content, url: responseUrl, path, error } of responses) {\n const resolvedUrl = responseUrl ?? path ?? '';\n const resources =\n error !== undefined || !content ? [] : await resourceExtractor.extract(content);\n const descriptor =\n error !== undefined || !content\n ? { url: resolvedUrl, error: error ?? 'Unknown error', resources }\n : await pageExtractor.extract(content);\n pageMetadatas.push({ ...descriptor, resources });\n }\n\n return pageMetadatas;\n}\n\nfunction normalizeLocalPath(value: string): string {\n if (value.startsWith('file://')) {\n return fileURLToPath(value);\n }\n\n return value;\n}\n\nasync function runFileCommand(paths: string[], options: { failsafe: boolean }): Promise<void> {\n if (options.failsafe && paths.length > MAX_FILES_FAILSAFE) {\n console.error(\n `\\n\u274C ${paths.length} files specified exceeds the safety limit of ${MAX_FILES_FAILSAFE}.`\n );\n console.error(` Pass --no-failsafe to bypass this check and process all files.`);\n process.exit(1);\n }\n\n if (!options.failsafe && paths.length > MAX_FILES_FAILSAFE) {\n console.error(\n `\\n\u26A0\uFE0F Failsafe bypassed: processing ${paths.length} files (limit is ${MAX_FILES_FAILSAFE}).`\n );\n }\n\n console.error(`\\n\u2705 Processing ${paths.length} file(s)...`);\n\n const fileFetcher = new FileFetcher();\n const normalizedPaths = paths.map((pathValue) => normalizeLocalPath(pathValue));\n const responses = await fileFetcher.fetchAll(normalizedPaths);\n const pageMetadatas = await buildPageMetadata(\n responses.map(({ path, content, error }) => ({ path, content, error }))\n );\n\n await printer.print(...pageMetadatas);\n}\n\nfunction isCliEntrypoint(): boolean {\n const invokedPath = process.argv[1];\n if (!invokedPath) {\n return false;\n }\n\n try {\n return realpathSync(fileURLToPath(import.meta.url)) === realpathSync(resolve(invokedPath));\n } catch {\n return false;\n }\n}\n\nexport async function runCli(argv: string[] = process.argv): Promise<void> {\n program.name(name).version(version, '-v, --version').description(description);\n\n program\n .addArgument(fileArg)\n .addOption(\n new Option('--no-failsafe', `bypass the ${MAX_FILES_FAILSAFE}-file limit safety check`)\n )\n .action(async (paths: string[], options: { failsafe: boolean }) => {\n try {\n await runFileCommand(paths, options);\n } catch (error) {\n console.error('\\n\u274C An error occurred:', error instanceof Error ? error.message : error);\n process.exit(1);\n }\n });\n\n // \u2500\u2500 fetch subcommand (remote URL mode only) \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n program\n .command('fetch')\n .description('fetch and extract resources from remote URL(s)')\n .addArgument(urlArg)\n .addOption(\n new Option(\n '--watch',\n 'keep running: SIGWINCH re-fetches after resize, Ctrl-D releases in-flight requests, Ctrl-C exits'\n )\n )\n .addOption(new Option('-A, --user-agent <value>', 'override the HTTP User-Agent header'))\n .addOption(\n new Option(\n '--allow-private-hosts',\n 'allow localhost/private-network targets (disabled by default for SSRF safety)'\n )\n )\n .action(\n async (\n urls: string[],\n options: { watch: boolean; userAgent?: string; allowPrivateHosts: boolean }\n ) => {\n try {\n const { validUrls, errors } = validateUrls(urls, {\n allowPrivateHosts: options.allowPrivateHosts,\n });\n\n if (errors.length > 0) {\n console.error('\\n\u274C URL Validation Errors:');\n errors.forEach(({ url: invalidUrl, error }) => {\n console.error(` - ${invalidUrl}: ${error}`);\n });\n }\n\n if (validUrls.length === 0) {\n console.error('\\n\u274C No valid URLs to process. Exiting.');\n process.exit(1);\n }\n\n console.error(`\\n\u2705 Processing ${validUrls.length} valid URL(s)...`);\n\n const pageFetcher = new PageFetcher(options.watch ? 0 : 10000, 2, options.userAgent);\n\n const execute = async (): Promise<void> => {\n const responses = await pageFetcher.fetchAll(validUrls);\n const pageMetadatas = await buildPageMetadata(responses);\n await printer.print(...pageMetadatas);\n };\n\n if (options.watch) {\n process.stdin.resume();\n process.on('SIGINT', () => process.exit(0));\n\n let activeExecution: Promise<void> | null = null;\n process.stdin.on('end', () => {\n activeExecution = null;\n });\n\n let winchTimer: ReturnType<typeof setTimeout> | null = null;\n process.on('SIGWINCH', () => {\n if (winchTimer !== null) clearTimeout(winchTimer);\n winchTimer = setTimeout(() => {\n winchTimer = null;\n activeExecution = execute().catch((err: unknown) => {\n console.error(\n '\\n\u274C An error occurred:',\n err instanceof Error ? err.message : err\n );\n });\n }, 150);\n });\n\n activeExecution = execute();\n await activeExecution;\n } else {\n await execute();\n }\n } catch (error) {\n console.error('\\n\u274C An error occurred:', error instanceof Error ? error.message : error);\n process.exit(1);\n }\n }\n );\n\n // \u2500\u2500 file subcommand (local filesystem access) \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n program\n .command('file')\n .description('extract resources from local file(s) via direct filesystem access')\n .addArgument(fileArg)\n .addOption(\n new Option('--no-failsafe', `bypass the ${MAX_FILES_FAILSAFE}-file limit safety check`)\n )\n .action(async (paths: string[], options: { failsafe: boolean }) => {\n try {\n await runFileCommand(paths, options);\n } catch (error) {\n console.error('\\n\u274C An error occurred:', error instanceof Error ? error.message : error);\n process.exit(1);\n }\n });\n\n await program.parseAsync(argv);\n}\n\nif (isCliEntrypoint()) {\n runCli().catch((error: unknown) => {\n console.error('\\n\u274C An error occurred:', error instanceof Error ? error.message : error);\n process.exit(1);\n });\n}\n", "export abstract class AbstractExtractor<V, R> {\n constructor(readonly name: string) {}\n abstract extract(value: V): Promise<R>;\n}\n", "import type { Page } from '../page/index.js';\nimport type { DOMResult } from '../page/index.js';\nimport { AbstractExtractor } from './AbstractExtractor.js';\n\nexport class PageExtractor extends AbstractExtractor<DOMResult, Page> {\n constructor() {\n super('page-extractor');\n }\n\n async extract(value: DOMResult): Promise<Page> {\n const {\n window: { document },\n url,\n } = value;\n return { title: document.title, url };\n }\n}\n", "/**\n * @license MIT\n * We are interested in visualising a page as a collection of tags.\n *\n * We wish to work with tags that can be compactly previewed on a webpage.\n * Here we must declare all of the element types that can be used to represent\n * a resource that can be hyperlinked off a webpage.\n */\ntype Tags = HTMLElementTagNameMap;\n\nexport const RESOURCE_DISPLAYABLE_KEYS = [\n 'id',\n 'innerText',\n 'textContent',\n 'class',\n 'ariaLabel',\n 'ariaDescription',\n 'alt',\n] as const;\n\nexport type DisplayableKey = (typeof RESOURCE_DISPLAYABLE_KEYS)[number];\n\nexport const RESOURCE_LINK_KEYS = ['href', 'data-src', 'target', 'action', 'src', 'url'] as const;\n\nexport type LinkKey = (typeof RESOURCE_LINK_KEYS)[number];\n\nexport type AttributeKey = DisplayableKey | LinkKey;\n\nexport type ResourceKey = { key: AttributeKey; value: string };\nexport type ResourceLink = { key: LinkKey; value: string };\n\nexport type ExternalResource = {\n text: ResourceKey;\n link: ResourceLink;\n};\n\nexport type Tag = keyof Tags;\n\nexport type Resource = HTMLElement & {\n [K in AttributeKey]?: string | null;\n};\n\nexport type ResourceByName<T extends keyof Tags> = Tags[T];\n\n// --- adapters ---\n\nconst readAttr = (element: Resource, key: AttributeKey): string | undefined => {\n const v = element.getAttribute(key);\n return v != null && v.trim() !== '' ? v : undefined;\n};\n\nexport function findResourceText(element: Resource): ResourceKey | undefined {\n for (const key of RESOURCE_DISPLAYABLE_KEYS) {\n const value = readAttr(element, key);\n if (value !== undefined) return { key, value };\n }\n return undefined;\n}\n\nexport function findResourceLink(element: Resource): ResourceLink | undefined {\n for (const key of RESOURCE_LINK_KEYS) {\n const value = readAttr(element, key);\n if (value !== undefined) return { key, value };\n }\n return undefined;\n}\n\nexport const isResourceKey = (key: string): key is AttributeKey =>\n (RESOURCE_DISPLAYABLE_KEYS as readonly string[]).includes(key) ||\n (RESOURCE_LINK_KEYS as readonly string[]).includes(key);\n", "import type { DOMResult } from '../page/index.js';\nimport {\n findResourceLink,\n findResourceText,\n type ExternalResource,\n type Resource,\n type Tag,\n} from '../resource.js';\nimport { AbstractExtractor } from './AbstractExtractor.js';\n\nexport class ResourceExtractor extends AbstractExtractor<DOMResult, ExternalResource[]> {\n constructor(private readonly tags: Tag[]) {\n super('page-extractor');\n }\n async extract(value: DOMResult): Promise<ExternalResource[]> {\n const { document } = value.window;\n return this.tags.flatMap((tag) =>\n Array.from(document.querySelectorAll<Resource>(tag)).flatMap((element) => {\n const link = findResourceLink(element);\n if (!link) return [];\n const text = findResourceText(element) ?? { key: 'src' as const, value: link.value };\n return [{ text, link }];\n })\n );\n }\n}\n", "import { parseHTML } from 'linkedom';\n\nconst MAX_HTML_BYTES = 2 * 1024 * 1024;\nconst ALLOWED_CONTENT_TYPES = ['text/html', 'application/xhtml+xml'];\n\ntype ParseHTMLResult = {\n document: Document;\n};\n\nexport interface DOMResult {\n window: { document: Document };\n url: string;\n}\n\ninterface PageResponse {\n url: string;\n content?: DOMResult;\n error?: string;\n}\n\nexport class PageFetcher {\n private readonly timeout: number;\n private readonly maxRetries: number;\n private readonly userAgent?: string;\n\n constructor(timeout = 10000, maxRetries = 2, userAgent?: string) {\n this.timeout = timeout;\n this.maxRetries = maxRetries;\n this.userAgent = userAgent;\n }\n\n private buildDOMResult(html: string, url: string): DOMResult {\n const { document } = parseHTML(html) as ParseHTMLResult;\n return { window: { document }, url };\n }\n\n private decodeHtml(buffer: ArrayBuffer, charset: string): string {\n try {\n return new TextDecoder(charset).decode(new Uint8Array(buffer));\n } catch {\n return new TextDecoder('utf-8').decode(new Uint8Array(buffer));\n }\n }\n\n private async fetchPage(url: string, retryCount = 0): Promise<PageResponse> {\n const controller = new AbortController();\n let timeoutId: ReturnType<typeof setTimeout> | null = null;\n\n try {\n if (this.timeout > 0) {\n timeoutId = setTimeout(() => {\n controller.abort(new Error('Request timeout'));\n }, this.timeout);\n }\n\n const headers = this.userAgent ? { 'user-agent': this.userAgent } : undefined;\n const content = await fetch(url, { headers, signal: controller.signal }).then(\n async (response) => {\n if (!response.ok) {\n throw new Error(`HTTP ${response.status} ${response.statusText}`.trim());\n }\n\n const contentType = (response.headers.get('content-type') ?? '').toLowerCase();\n const isAllowedContentType = ALLOWED_CONTENT_TYPES.some((allowedType) =>\n contentType.includes(allowedType)\n );\n if (!isAllowedContentType) {\n throw new Error(`Unsupported content type: ${contentType || 'unknown'}`);\n }\n\n const contentLengthHeader = response.headers.get('content-length');\n const contentLength = contentLengthHeader ? Number(contentLengthHeader) : Number.NaN;\n if (Number.isFinite(contentLength) && contentLength > MAX_HTML_BYTES) {\n throw new Error(`Response exceeds max allowed size (${MAX_HTML_BYTES} bytes)`);\n }\n\n const buffer = await response.arrayBuffer();\n if (buffer.byteLength > MAX_HTML_BYTES) {\n throw new Error(`Response exceeds max allowed size (${MAX_HTML_BYTES} bytes)`);\n }\n\n const charsetMatch = /charset=([^\\s;]+)/i.exec(contentType);\n const html = this.decodeHtml(buffer, charsetMatch?.[1] ?? 'utf-8');\n return this.buildDOMResult(html, url);\n }\n );\n\n return { url, content };\n } catch (error) {\n const abortTimeout = error instanceof Error && error.name === 'AbortError';\n const message = abortTimeout\n ? 'Request timeout'\n : error instanceof Error\n ? error.message\n : 'Unknown error';\n\n // Retry logic for transient errors\n if (retryCount < this.maxRetries && this.isRetryableError(message)) {\n process.stderr.write(`Retrying ${url} (attempt ${retryCount + 1}/${this.maxRetries})...\\n`);\n await this.delay(1000 * (retryCount + 1)); // Exponential backoff\n return this.fetchPage(url, retryCount + 1);\n }\n\n return { url, error: `Failed to fetch: ${message}` };\n } finally {\n if (timeoutId !== null) {\n clearTimeout(timeoutId);\n }\n }\n }\n\n private isRetryableError(message: string): boolean {\n const retryablePatterns = [/timeout/i, /ECONNRESET/i, /ETIMEDOUT/i, /ENOTFOUND/i, /network/i];\n return retryablePatterns.some((pattern) => pattern.test(message));\n }\n\n private delay(ms: number): Promise<void> {\n return new Promise((resolve) => setTimeout(resolve, ms));\n }\n\n async fetchAll(urls: string[]): Promise<PageResponse[]> {\n const responses = await Promise.all(urls.map((url) => this.fetchPage(url)));\n return responses.filter((response) => response.content !== undefined || response.error);\n }\n}\n", "import { readFile } from 'node:fs/promises';\nimport { parseHTML } from 'linkedom';\n\nimport type { DOMResult } from './PageFetcher.js';\n\nexport const MAX_FILES_FAILSAFE = 254;\n\ntype ParseHTMLResult = {\n document: Document;\n};\n\nexport interface FileResponse {\n path: string;\n content?: DOMResult;\n error?: string;\n}\n\nexport class FileFetcher {\n private buildDOMResult(html: string, filePath: string): DOMResult {\n const { document } = parseHTML(html) as ParseHTMLResult;\n return { window: { document }, url: `file://${filePath}` };\n }\n\n async fetchFile(filePath: string): Promise<FileResponse> {\n try {\n // filePath is supplied directly by the CLI user, not derived from network input.\n // eslint-disable-next-line security/detect-non-literal-fs-filename\n const html = await readFile(filePath, 'utf-8');\n return { path: filePath, content: this.buildDOMResult(html, filePath) };\n } catch (error) {\n return {\n path: filePath,\n error: error instanceof Error ? error.message : 'Unknown error',\n };\n }\n }\n\n async fetchAll(filePaths: string[]): Promise<FileResponse[]> {\n return Promise.all(filePaths.map((p) => this.fetchFile(p)));\n }\n}\n", "import type { PageMetadata } from '../page/index.js';\n\nexport abstract class AbstractResourcePrinter {\n constructor() {}\n abstract print(...pages: PageMetadata[]): void | Promise<void>;\n}\n", "import type { PageMetadata } from '../page/index.js';\nimport { AbstractResourcePrinter } from './AbstractResourcePrinter.js';\n\nexport class JSONStylePrinter extends AbstractResourcePrinter {\n print(...pages: PageMetadata[]): void | Promise<void> {\n const json = JSON.stringify(pages);\n process.stdout.write(json + '\\n');\n }\n}\n", "/**\n * Security utilities for URL validation and sanitization\n */\n\nimport { isIP } from 'node:net';\n\nconst ALLOWED_PROTOCOLS = ['http:', 'https:'];\nconst MAX_URL_LENGTH = 2048;\nconst SUSPICIOUS_PATTERNS = [\n /javascript:/i,\n /data:/i,\n /vbscript:/i,\n /<script/i,\n /on\\w+=/i, // Event handlers like onclick=\n];\n\nexport interface ValidationResult {\n isValid: boolean;\n error?: string;\n sanitizedUrl?: string;\n}\n\nexport interface UrlValidationOptions {\n allowPrivateHosts?: boolean;\n}\n\nfunction isPrivateHostname(hostname: string): boolean {\n const normalized = hostname.toLowerCase();\n\n if (normalized === 'localhost' || normalized.endsWith('.localhost')) {\n return true;\n }\n\n const ipType = isIP(normalized);\n if (ipType === 0) {\n return false;\n }\n\n if (ipType === 4) {\n const octets = normalized.split('.').map(Number);\n if (octets.length !== 4 || octets.some((octet) => Number.isNaN(octet))) {\n return false;\n }\n\n const [a, b] = octets;\n return (\n a === 10 ||\n a === 127 ||\n (a === 169 && b === 254) ||\n (a === 172 && b >= 16 && b <= 31) ||\n (a === 192 && b === 168) ||\n a === 0\n );\n }\n\n // IPv6 private/internal ranges.\n return (\n normalized === '::1' ||\n normalized === '::' ||\n normalized.startsWith('fc') ||\n normalized.startsWith('fd') ||\n normalized.startsWith('fe8') ||\n normalized.startsWith('fe9') ||\n normalized.startsWith('fea') ||\n normalized.startsWith('feb')\n );\n}\n\n/**\n * Validates a URL for security concerns\n * @param url - The URL to validate\n * @returns ValidationResult object with validation status\n */\nexport function validateUrl(url: string, options: UrlValidationOptions = {}): ValidationResult {\n // Check if URL is empty or whitespace\n if (!url || !url.trim()) {\n return {\n isValid: false,\n error: 'URL cannot be empty',\n };\n }\n\n const trimmedUrl = url.trim();\n\n // Check URL length to prevent DoS\n if (trimmedUrl.length > MAX_URL_LENGTH) {\n return {\n isValid: false,\n error: `URL exceeds maximum length of ${MAX_URL_LENGTH} characters`,\n };\n }\n\n // Check for suspicious patterns\n for (const pattern of SUSPICIOUS_PATTERNS) {\n if (pattern.test(trimmedUrl)) {\n return {\n isValid: false,\n error: 'URL contains suspicious patterns',\n };\n }\n }\n\n // Parse the URL\n let parsedUrl: URL;\n try {\n parsedUrl = new URL(trimmedUrl);\n } catch {\n return {\n isValid: false,\n error: 'Invalid URL format',\n };\n }\n\n // Check protocol\n if (!ALLOWED_PROTOCOLS.includes(parsedUrl.protocol)) {\n return {\n isValid: false,\n error: `Protocol ${parsedUrl.protocol} is not allowed. Allowed protocols: ${ALLOWED_PROTOCOLS.join(', ')}`,\n };\n }\n\n if (parsedUrl.username || parsedUrl.password) {\n return {\n isValid: false,\n error: 'URLs with embedded credentials are not allowed',\n };\n }\n\n if (!options.allowPrivateHosts && isPrivateHostname(parsedUrl.hostname)) {\n return {\n isValid: false,\n error:\n 'Private or loopback hostnames are blocked by default. Use --allow-private-hosts if you trust the target.',\n };\n }\n\n return {\n isValid: true,\n sanitizedUrl: parsedUrl.toString(),\n };\n}\n\n/**\n * Validates an array of URLs\n * @param urls - Array of URLs to validate\n * @returns Object with valid URLs and errors\n */\nexport function validateUrls(\n urls: string[],\n options: UrlValidationOptions = {}\n): {\n validUrls: string[];\n errors: Array<{ url: string; error: string }>;\n} {\n const validUrls: string[] = [];\n const errors: Array<{ url: string; error: string }> = [];\n\n for (const url of urls) {\n const result = validateUrl(url, options);\n if (result.isValid && result.sanitizedUrl) {\n validUrls.push(result.sanitizedUrl);\n } else {\n errors.push({\n url,\n error: result.error || 'Unknown validation error',\n });\n }\n }\n\n return { validUrls, errors };\n}\n\n/**\n * Rate limiter to prevent abuse\n */\nexport class RateLimiter {\n private requests: number[] = [];\n private readonly maxRequests: number;\n private readonly windowMs: number;\n\n constructor(maxRequests = 10, windowMs = 60000) {\n this.maxRequests = maxRequests;\n this.windowMs = windowMs;\n }\n\n /**\n * Check if a request is allowed under rate limiting\n * @returns true if request is allowed, false otherwise\n */\n public isAllowed(): boolean {\n const now = Date.now();\n\n // Remove old requests outside the time window\n this.requests = this.requests.filter((time) => now - time < this.windowMs);\n\n if (this.requests.length >= this.maxRequests) {\n return false;\n }\n\n this.requests.push(now);\n return true;\n }\n\n /**\n * Get remaining requests in current window\n */\n public getRemainingRequests(): number {\n const now = Date.now();\n this.requests = this.requests.filter((time) => now - time < this.windowMs);\n return Math.max(0, this.maxRequests - this.requests.length);\n }\n}\n\n/**\n * Sanitizes HTML content to prevent XSS attacks\n * @param text - Text to sanitize\n * @returns Sanitized text\n */\nexport function sanitizeText(text: string): string {\n if (!text) return '';\n\n return text\n .replace(/</g, '<')\n .replace(/>/g, '>')\n .replace(/\"/g, '"')\n .replace(/'/g, ''')\n .replace(/\\//g, '/');\n}\n"],
|
|
5
|
+
"mappings": ";;;AACA,SAAS,SAAS,gBAAgB,cAAc;AAChD,SAAS,qBAAqB;AAC9B,SAAS,eAAe;AACxB,SAAS,qBAAqB;AAC9B,SAAS,oBAAoB;;;ACLtB,IAAe,oBAAf,MAAuC;AAAA,EAC5C,YAAqBA,OAAc;AAAd,gBAAAA;AAAA,EAAe;AAAA,EAAf;AAEvB;;;ACCO,IAAM,gBAAN,cAA4B,kBAAmC;AAAA,EACpE,cAAc;AACZ,UAAM,gBAAgB;AAAA,EACxB;AAAA,EAEA,MAAM,QAAQ,OAAiC;AAC7C,UAAM;AAAA,MACJ,QAAQ,EAAE,SAAS;AAAA,MACnB;AAAA,IACF,IAAI;AACJ,WAAO,EAAE,OAAO,SAAS,OAAO,IAAI;AAAA,EACtC;AACF;;;ACNO,IAAM,4BAA4B;AAAA,EACvC;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AACF;AAIO,IAAM,qBAAqB,CAAC,QAAQ,YAAY,UAAU,UAAU,OAAO,KAAK;AAwBvF,IAAM,WAAW,CAAC,SAAmB,QAA0C;AAC7E,QAAM,IAAI,QAAQ,aAAa,GAAG;AAClC,SAAO,KAAK,QAAQ,EAAE,KAAK,MAAM,KAAK,IAAI;AAC5C;AAEO,SAAS,iBAAiB,SAA4C;AAC3E,aAAW,OAAO,2BAA2B;AAC3C,UAAM,QAAQ,SAAS,SAAS,GAAG;AACnC,QAAI,UAAU,OAAW,QAAO,EAAE,KAAK,MAAM;AAAA,EAC/C;AACA,SAAO;AACT;AAEO,SAAS,iBAAiB,SAA6C;AAC5E,aAAW,OAAO,oBAAoB;AACpC,UAAM,QAAQ,SAAS,SAAS,GAAG;AACnC,QAAI,UAAU,OAAW,QAAO,EAAE,KAAK,MAAM;AAAA,EAC/C;AACA,SAAO;AACT;;;ACvDO,IAAM,oBAAN,cAAgC,kBAAiD;AAAA,EACtF,YAA6B,MAAa;AACxC,UAAM,gBAAgB;AADK;AAAA,EAE7B;AAAA,EAF6B;AAAA,EAG7B,MAAM,QAAQ,OAA+C;AAC3D,UAAM,EAAE,SAAS,IAAI,MAAM;AAC3B,WAAO,KAAK,KAAK;AAAA,MAAQ,CAAC,QACxB,MAAM,KAAK,SAAS,iBAA2B,GAAG,CAAC,EAAE,QAAQ,CAAC,YAAY;AACxE,cAAM,OAAO,iBAAiB,OAAO;AACrC,YAAI,CAAC,KAAM,QAAO,CAAC;AACnB,cAAM,OAAO,iBAAiB,OAAO,KAAK,EAAE,KAAK,OAAgB,OAAO,KAAK,MAAM;AACnF,eAAO,CAAC,EAAE,MAAM,KAAK,CAAC;AAAA,MACxB,CAAC;AAAA,IACH;AAAA,EACF;AACF;;;ACzBA,SAAS,iBAAiB;AAE1B,IAAM,iBAAiB,IAAI,OAAO;AAClC,IAAM,wBAAwB,CAAC,aAAa,uBAAuB;AAiB5D,IAAM,cAAN,MAAkB;AAAA,EACN;AAAA,EACA;AAAA,EACA;AAAA,EAEjB,YAAY,UAAU,KAAO,aAAa,GAAG,WAAoB;AAC/D,SAAK,UAAU;AACf,SAAK,aAAa;AAClB,SAAK,YAAY;AAAA,EACnB;AAAA,EAEQ,eAAe,MAAc,KAAwB;AAC3D,UAAM,EAAE,SAAS,IAAI,UAAU,IAAI;AACnC,WAAO,EAAE,QAAQ,EAAE,SAAS,GAAG,IAAI;AAAA,EACrC;AAAA,EAEQ,WAAW,QAAqB,SAAyB;AAC/D,QAAI;AACF,aAAO,IAAI,YAAY,OAAO,EAAE,OAAO,IAAI,WAAW,MAAM,CAAC;AAAA,IAC/D,QAAQ;AACN,aAAO,IAAI,YAAY,OAAO,EAAE,OAAO,IAAI,WAAW,MAAM,CAAC;AAAA,IAC/D;AAAA,EACF;AAAA,EAEA,MAAc,UAAU,KAAa,aAAa,GAA0B;AAC1E,UAAM,aAAa,IAAI,gBAAgB;AACvC,QAAI,YAAkD;AAEtD,QAAI;AACF,UAAI,KAAK,UAAU,GAAG;AACpB,oBAAY,WAAW,MAAM;AAC3B,qBAAW,MAAM,IAAI,MAAM,iBAAiB,CAAC;AAAA,QAC/C,GAAG,KAAK,OAAO;AAAA,MACjB;AAEA,YAAM,UAAU,KAAK,YAAY,EAAE,cAAc,KAAK,UAAU,IAAI;AACpE,YAAM,UAAU,MAAM,MAAM,KAAK,EAAE,SAAS,QAAQ,WAAW,OAAO,CAAC,EAAE;AAAA,QACvE,OAAO,aAAa;AAClB,cAAI,CAAC,SAAS,IAAI;AAChB,kBAAM,IAAI,MAAM,QAAQ,SAAS,MAAM,IAAI,SAAS,UAAU,GAAG,KAAK,CAAC;AAAA,UACzE;AAEA,gBAAM,eAAe,SAAS,QAAQ,IAAI,cAAc,KAAK,IAAI,YAAY;AAC7E,gBAAM,uBAAuB,sBAAsB;AAAA,YAAK,CAAC,gBACvD,YAAY,SAAS,WAAW;AAAA,UAClC;AACA,cAAI,CAAC,sBAAsB;AACzB,kBAAM,IAAI,MAAM,6BAA6B,eAAe,SAAS,EAAE;AAAA,UACzE;AAEA,gBAAM,sBAAsB,SAAS,QAAQ,IAAI,gBAAgB;AACjE,gBAAM,gBAAgB,sBAAsB,OAAO,mBAAmB,IAAI,OAAO;AACjF,cAAI,OAAO,SAAS,aAAa,KAAK,gBAAgB,gBAAgB;AACpE,kBAAM,IAAI,MAAM,sCAAsC,cAAc,SAAS;AAAA,UAC/E;AAEA,gBAAM,SAAS,MAAM,SAAS,YAAY;AAC1C,cAAI,OAAO,aAAa,gBAAgB;AACtC,kBAAM,IAAI,MAAM,sCAAsC,cAAc,SAAS;AAAA,UAC/E;AAEA,gBAAM,eAAe,qBAAqB,KAAK,WAAW;AAC1D,gBAAM,OAAO,KAAK,WAAW,QAAQ,eAAe,CAAC,KAAK,OAAO;AACjE,iBAAO,KAAK,eAAe,MAAM,GAAG;AAAA,QACtC;AAAA,MACF;AAEA,aAAO,EAAE,KAAK,QAAQ;AAAA,IACxB,SAAS,OAAO;AACd,YAAM,eAAe,iBAAiB,SAAS,MAAM,SAAS;AAC9D,YAAM,UAAU,eACZ,oBACA,iBAAiB,QACf,MAAM,UACN;AAGN,UAAI,aAAa,KAAK,cAAc,KAAK,iBAAiB,OAAO,GAAG;AAClE,gBAAQ,OAAO,MAAM,YAAY,GAAG,aAAa,aAAa,CAAC,IAAI,KAAK,UAAU;AAAA,CAAQ;AAC1F,cAAM,KAAK,MAAM,OAAQ,aAAa,EAAE;AACxC,eAAO,KAAK,UAAU,KAAK,aAAa,CAAC;AAAA,MAC3C;AAEA,aAAO,EAAE,KAAK,OAAO,oBAAoB,OAAO,GAAG;AAAA,IACrD,UAAE;AACA,UAAI,cAAc,MAAM;AACtB,qBAAa,SAAS;AAAA,MACxB;AAAA,IACF;AAAA,EACF;AAAA,EAEQ,iBAAiB,SAA0B;AACjD,UAAM,oBAAoB,CAAC,YAAY,eAAe,cAAc,cAAc,UAAU;AAC5F,WAAO,kBAAkB,KAAK,CAAC,YAAY,QAAQ,KAAK,OAAO,CAAC;AAAA,EAClE;AAAA,EAEQ,MAAM,IAA2B;AACvC,WAAO,IAAI,QAAQ,CAACC,aAAY,WAAWA,UAAS,EAAE,CAAC;AAAA,EACzD;AAAA,EAEA,MAAM,SAAS,MAAyC;AACtD,UAAM,YAAY,MAAM,QAAQ,IAAI,KAAK,IAAI,CAAC,QAAQ,KAAK,UAAU,GAAG,CAAC,CAAC;AAC1E,WAAO,UAAU,OAAO,CAAC,aAAa,SAAS,YAAY,UAAa,SAAS,KAAK;AAAA,EACxF;AACF;;;AC5HA,SAAS,gBAAgB;AACzB,SAAS,aAAAC,kBAAiB;AAInB,IAAM,qBAAqB;AAY3B,IAAM,cAAN,MAAkB;AAAA,EACf,eAAe,MAAc,UAA6B;AAChE,UAAM,EAAE,SAAS,IAAIA,WAAU,IAAI;AACnC,WAAO,EAAE,QAAQ,EAAE,SAAS,GAAG,KAAK,UAAU,QAAQ,GAAG;AAAA,EAC3D;AAAA,EAEA,MAAM,UAAU,UAAyC;AACvD,QAAI;AAGF,YAAM,OAAO,MAAM,SAAS,UAAU,OAAO;AAC7C,aAAO,EAAE,MAAM,UAAU,SAAS,KAAK,eAAe,MAAM,QAAQ,EAAE;AAAA,IACxE,SAAS,OAAO;AACd,aAAO;AAAA,QACL,MAAM;AAAA,QACN,OAAO,iBAAiB,QAAQ,MAAM,UAAU;AAAA,MAClD;AAAA,IACF;AAAA,EACF;AAAA,EAEA,MAAM,SAAS,WAA8C;AAC3D,WAAO,QAAQ,IAAI,UAAU,IAAI,CAAC,MAAM,KAAK,UAAU,CAAC,CAAC,CAAC;AAAA,EAC5D;AACF;;;ACtCO,IAAe,0BAAf,MAAuC;AAAA,EAC5C,cAAc;AAAA,EAAC;AAEjB;;;ACFO,IAAM,mBAAN,cAA+B,wBAAwB;AAAA,EAC5D,SAAS,OAA6C;AACpD,UAAM,OAAO,KAAK,UAAU,KAAK;AACjC,YAAQ,OAAO,MAAM,OAAO,IAAI;AAAA,EAClC;AACF;;;ACJA,SAAS,YAAY;AAErB,IAAM,oBAAoB,CAAC,SAAS,QAAQ;AAC5C,IAAM,iBAAiB;AACvB,IAAM,sBAAsB;AAAA,EAC1B;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA;AACF;AAYA,SAAS,kBAAkB,UAA2B;AACpD,QAAM,aAAa,SAAS,YAAY;AAExC,MAAI,eAAe,eAAe,WAAW,SAAS,YAAY,GAAG;AACnE,WAAO;AAAA,EACT;AAEA,QAAM,SAAS,KAAK,UAAU;AAC9B,MAAI,WAAW,GAAG;AAChB,WAAO;AAAA,EACT;AAEA,MAAI,WAAW,GAAG;AAChB,UAAM,SAAS,WAAW,MAAM,GAAG,EAAE,IAAI,MAAM;AAC/C,QAAI,OAAO,WAAW,KAAK,OAAO,KAAK,CAAC,UAAU,OAAO,MAAM,KAAK,CAAC,GAAG;AACtE,aAAO;AAAA,IACT;AAEA,UAAM,CAAC,GAAG,CAAC,IAAI;AACf,WACE,MAAM,MACN,MAAM,OACL,MAAM,OAAO,MAAM,OACnB,MAAM,OAAO,KAAK,MAAM,KAAK,MAC7B,MAAM,OAAO,MAAM,OACpB,MAAM;AAAA,EAEV;AAGA,SACE,eAAe,SACf,eAAe,QACf,WAAW,WAAW,IAAI,KAC1B,WAAW,WAAW,IAAI,KAC1B,WAAW,WAAW,KAAK,KAC3B,WAAW,WAAW,KAAK,KAC3B,WAAW,WAAW,KAAK,KAC3B,WAAW,WAAW,KAAK;AAE/B;AAOO,SAAS,YAAY,KAAa,UAAgC,CAAC,GAAqB;AAE7F,MAAI,CAAC,OAAO,CAAC,IAAI,KAAK,GAAG;AACvB,WAAO;AAAA,MACL,SAAS;AAAA,MACT,OAAO;AAAA,IACT;AAAA,EACF;AAEA,QAAM,aAAa,IAAI,KAAK;AAG5B,MAAI,WAAW,SAAS,gBAAgB;AACtC,WAAO;AAAA,MACL,SAAS;AAAA,MACT,OAAO,iCAAiC,cAAc;AAAA,IACxD;AAAA,EACF;AAGA,aAAW,WAAW,qBAAqB;AACzC,QAAI,QAAQ,KAAK,UAAU,GAAG;AAC5B,aAAO;AAAA,QACL,SAAS;AAAA,QACT,OAAO;AAAA,MACT;AAAA,IACF;AAAA,EACF;AAGA,MAAI;AACJ,MAAI;AACF,gBAAY,IAAI,IAAI,UAAU;AAAA,EAChC,QAAQ;AACN,WAAO;AAAA,MACL,SAAS;AAAA,MACT,OAAO;AAAA,IACT;AAAA,EACF;AAGA,MAAI,CAAC,kBAAkB,SAAS,UAAU,QAAQ,GAAG;AACnD,WAAO;AAAA,MACL,SAAS;AAAA,MACT,OAAO,YAAY,UAAU,QAAQ,uCAAuC,kBAAkB,KAAK,IAAI,CAAC;AAAA,IAC1G;AAAA,EACF;AAEA,MAAI,UAAU,YAAY,UAAU,UAAU;AAC5C,WAAO;AAAA,MACL,SAAS;AAAA,MACT,OAAO;AAAA,IACT;AAAA,EACF;AAEA,MAAI,CAAC,QAAQ,qBAAqB,kBAAkB,UAAU,QAAQ,GAAG;AACvE,WAAO;AAAA,MACL,SAAS;AAAA,MACT,OACE;AAAA,IACJ;AAAA,EACF;AAEA,SAAO;AAAA,IACL,SAAS;AAAA,IACT,cAAc,UAAU,SAAS;AAAA,EACnC;AACF;AAOO,SAAS,aACd,MACA,UAAgC,CAAC,GAIjC;AACA,QAAM,YAAsB,CAAC;AAC7B,QAAM,SAAgD,CAAC;AAEvD,aAAW,OAAO,MAAM;AACtB,UAAM,SAAS,YAAY,KAAK,OAAO;AACvC,QAAI,OAAO,WAAW,OAAO,cAAc;AACzC,gBAAU,KAAK,OAAO,YAAY;AAAA,IACpC,OAAO;AACL,aAAO,KAAK;AAAA,QACV;AAAA,QACA,OAAO,OAAO,SAAS;AAAA,MACzB,CAAC;AAAA,IACH;AAAA,EACF;AAEA,SAAO,EAAE,WAAW,OAAO;AAC7B;;;AT9JA,IAAMC,WAAU,cAAc,YAAY,GAAG;AAC7C,IAAM,MAAMA,SAAQ,iBAAiB;AAMrC,IAAM,EAAE,aAAa,MAAM,QAAQ,IAAI;AAEvC,IAAM,UAAU,IAAI,QAAQ;AAE5B,IAAM,SAAS,eAAe,YAAY,oCAAoC;AAC9E,IAAM,UAAU,eAAe,cAAc,kCAAkC;AAG/E,IAAM,gBAAgB,IAAI,cAAc;AACxC,IAAM,oBAAoB,IAAI,kBAAkB,CAAC,KAAK,QAAQ,QAAQ,SAAS,QAAQ,CAAC;AACxF,IAAM,UAAU,IAAI,iBAAiB;AAErC,eAAe,kBACb,WAMyB;AACzB,QAAM,gBAAgC,CAAC;AAEvC,aAAW,EAAE,SAAS,KAAK,aAAa,MAAM,MAAM,KAAK,WAAW;AAClE,UAAM,cAAc,eAAe,QAAQ;AAC3C,UAAM,YACJ,UAAU,UAAa,CAAC,UAAU,CAAC,IAAI,MAAM,kBAAkB,QAAQ,OAAO;AAChF,UAAM,aACJ,UAAU,UAAa,CAAC,UACpB,EAAE,KAAK,aAAa,OAAO,SAAS,iBAAiB,UAAU,IAC/D,MAAM,cAAc,QAAQ,OAAO;AACzC,kBAAc,KAAK,EAAE,GAAG,YAAY,UAAU,CAAC;AAAA,EACjD;AAEA,SAAO;AACT;AAEA,SAAS,mBAAmB,OAAuB;AACjD,MAAI,MAAM,WAAW,SAAS,GAAG;AAC/B,WAAO,cAAc,KAAK;AAAA,EAC5B;AAEA,SAAO;AACT;AAEA,eAAe,eAAe,OAAiB,SAA+C;AAC5F,MAAI,QAAQ,YAAY,MAAM,SAAS,oBAAoB;AACzD,YAAQ;AAAA,MACN;AAAA,SAAO,MAAM,MAAM,gDAAgD,kBAAkB;AAAA,IACvF;AACA,YAAQ,MAAM,mEAAmE;AACjF,YAAQ,KAAK,CAAC;AAAA,EAChB;AAEA,MAAI,CAAC,QAAQ,YAAY,MAAM,SAAS,oBAAoB;AAC1D,YAAQ;AAAA,MACN;AAAA,8CAAuC,MAAM,MAAM,oBAAoB,kBAAkB;AAAA,IAC3F;AAAA,EACF;AAEA,UAAQ,MAAM;AAAA,oBAAkB,MAAM,MAAM,aAAa;AAEzD,QAAM,cAAc,IAAI,YAAY;AACpC,QAAM,kBAAkB,MAAM,IAAI,CAAC,cAAc,mBAAmB,SAAS,CAAC;AAC9E,QAAM,YAAY,MAAM,YAAY,SAAS,eAAe;AAC5D,QAAM,gBAAgB,MAAM;AAAA,IAC1B,UAAU,IAAI,CAAC,EAAE,MAAM,SAAS,MAAM,OAAO,EAAE,MAAM,SAAS,MAAM,EAAE;AAAA,EACxE;AAEA,QAAM,QAAQ,MAAM,GAAG,aAAa;AACtC;AAEA,SAAS,kBAA2B;AAClC,QAAM,cAAc,QAAQ,KAAK,CAAC;AAClC,MAAI,CAAC,aAAa;AAChB,WAAO;AAAA,EACT;AAEA,MAAI;AACF,WAAO,aAAa,cAAc,YAAY,GAAG,CAAC,MAAM,aAAa,QAAQ,WAAW,CAAC;AAAA,EAC3F,QAAQ;AACN,WAAO;AAAA,EACT;AACF;AAEA,eAAsB,OAAO,OAAiB,QAAQ,MAAqB;AACzE,UAAQ,KAAK,IAAI,EAAE,QAAQ,SAAS,eAAe,EAAE,YAAY,WAAW;AAE5E,UACG,YAAY,OAAO,EACnB;AAAA,IACC,IAAI,OAAO,iBAAiB,cAAc,kBAAkB,0BAA0B;AAAA,EACxF,EACC,OAAO,OAAO,OAAiB,YAAmC;AACjE,QAAI;AACF,YAAM,eAAe,OAAO,OAAO;AAAA,IACrC,SAAS,OAAO;AACd,cAAQ,MAAM,+BAA0B,iBAAiB,QAAQ,MAAM,UAAU,KAAK;AACtF,cAAQ,KAAK,CAAC;AAAA,IAChB;AAAA,EACF,CAAC;AAGH,UACG,QAAQ,OAAO,EACf,YAAY,gDAAgD,EAC5D,YAAY,MAAM,EAClB;AAAA,IACC,IAAI;AAAA,MACF;AAAA,MACA;AAAA,IACF;AAAA,EACF,EACC,UAAU,IAAI,OAAO,4BAA4B,qCAAqC,CAAC,EACvF;AAAA,IACC,IAAI;AAAA,MACF;AAAA,MACA;AAAA,IACF;AAAA,EACF,EACC;AAAA,IACC,OACE,MACA,YACG;AACH,UAAI;AACF,cAAM,EAAE,WAAW,OAAO,IAAI,aAAa,MAAM;AAAA,UAC/C,mBAAmB,QAAQ;AAAA,QAC7B,CAAC;AAED,YAAI,OAAO,SAAS,GAAG;AACrB,kBAAQ,MAAM,iCAA4B;AAC1C,iBAAO,QAAQ,CAAC,EAAE,KAAK,YAAY,MAAM,MAAM;AAC7C,oBAAQ,MAAM,OAAO,UAAU,KAAK,KAAK,EAAE;AAAA,UAC7C,CAAC;AAAA,QACH;AAEA,YAAI,UAAU,WAAW,GAAG;AAC1B,kBAAQ,MAAM,6CAAwC;AACtD,kBAAQ,KAAK,CAAC;AAAA,QAChB;AAEA,gBAAQ,MAAM;AAAA,oBAAkB,UAAU,MAAM,kBAAkB;AAElE,cAAM,cAAc,IAAI,YAAY,QAAQ,QAAQ,IAAI,KAAO,GAAG,QAAQ,SAAS;AAEnF,cAAM,UAAU,YAA2B;AACzC,gBAAM,YAAY,MAAM,YAAY,SAAS,SAAS;AACtD,gBAAM,gBAAgB,MAAM,kBAAkB,SAAS;AACvD,gBAAM,QAAQ,MAAM,GAAG,aAAa;AAAA,QACtC;AAEA,YAAI,QAAQ,OAAO;AACjB,kBAAQ,MAAM,OAAO;AACrB,kBAAQ,GAAG,UAAU,MAAM,QAAQ,KAAK,CAAC,CAAC;AAE1C,cAAI,kBAAwC;AAC5C,kBAAQ,MAAM,GAAG,OAAO,MAAM;AAC5B,8BAAkB;AAAA,UACpB,CAAC;AAED,cAAI,aAAmD;AACvD,kBAAQ,GAAG,YAAY,MAAM;AAC3B,gBAAI,eAAe,KAAM,cAAa,UAAU;AAChD,yBAAa,WAAW,MAAM;AAC5B,2BAAa;AACb,gCAAkB,QAAQ,EAAE,MAAM,CAAC,QAAiB;AAClD,wBAAQ;AAAA,kBACN;AAAA,kBACA,eAAe,QAAQ,IAAI,UAAU;AAAA,gBACvC;AAAA,cACF,CAAC;AAAA,YACH,GAAG,GAAG;AAAA,UACR,CAAC;AAED,4BAAkB,QAAQ;AAC1B,gBAAM;AAAA,QACR,OAAO;AACL,gBAAM,QAAQ;AAAA,QAChB;AAAA,MACF,SAAS,OAAO;AACd,gBAAQ,MAAM,+BAA0B,iBAAiB,QAAQ,MAAM,UAAU,KAAK;AACtF,gBAAQ,KAAK,CAAC;AAAA,MAChB;AAAA,IACF;AAAA,EACF;AAGF,UACG,QAAQ,MAAM,EACd,YAAY,mEAAmE,EAC/E,YAAY,OAAO,EACnB;AAAA,IACC,IAAI,OAAO,iBAAiB,cAAc,kBAAkB,0BAA0B;AAAA,EACxF,EACC,OAAO,OAAO,OAAiB,YAAmC;AACjE,QAAI;AACF,YAAM,eAAe,OAAO,OAAO;AAAA,IACrC,SAAS,OAAO;AACd,cAAQ,MAAM,+BAA0B,iBAAiB,QAAQ,MAAM,UAAU,KAAK;AACtF,cAAQ,KAAK,CAAC;AAAA,IAChB;AAAA,EACF,CAAC;AAEH,QAAM,QAAQ,WAAW,IAAI;AAC/B;AAEA,IAAI,gBAAgB,GAAG;AACrB,SAAO,EAAE,MAAM,CAAC,UAAmB;AACjC,YAAQ,MAAM,+BAA0B,iBAAiB,QAAQ,MAAM,UAAU,KAAK;AACtF,YAAQ,KAAK,CAAC;AAAA,EAChB,CAAC;AACH;",
|
|
6
6
|
"names": ["name", "resolve", "parseHTML", "require"]
|
|
7
7
|
}
|
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "pagerts",
|
|
3
3
|
"description": "A tool for viewing external relations in a webpage",
|
|
4
|
-
"version": "1.5.
|
|
4
|
+
"version": "1.5.5",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "main.js",
|
|
7
7
|
"bin": {
|
|
@@ -43,18 +43,18 @@
|
|
|
43
43
|
},
|
|
44
44
|
"homepage": "https://github.com/akinevz2/pagerts",
|
|
45
45
|
"dependencies": {
|
|
46
|
-
"@exodus/bytes": "^1.15.0",
|
|
47
46
|
"commander": "^14.0.3",
|
|
48
47
|
"linkedom": "^0.18.9"
|
|
49
48
|
},
|
|
50
49
|
"devDependencies": {
|
|
50
|
+
"@eslint/js": "^9.39.1",
|
|
51
51
|
"@swc/core": "^1.15.33",
|
|
52
52
|
"@swc/jest": "^0.2.39",
|
|
53
53
|
"@types/jest": "^30.0.0",
|
|
54
54
|
"@types/node": "^25.8.0",
|
|
55
55
|
"@typescript-eslint/eslint-plugin": "^8.20.0",
|
|
56
56
|
"@typescript-eslint/parser": "^8.20.0",
|
|
57
|
-
"esbuild": "^0.
|
|
57
|
+
"esbuild": "^0.28.1",
|
|
58
58
|
"eslint": "^9.18.0",
|
|
59
59
|
"eslint-config-prettier": "^9.1.0",
|
|
60
60
|
"eslint-plugin-security": "^3.0.1",
|
|
@@ -66,6 +66,7 @@
|
|
|
66
66
|
"overrides": {
|
|
67
67
|
"babel-plugin-istanbul": "^8.0.0",
|
|
68
68
|
"test-exclude": "^8.0.0",
|
|
69
|
-
"glob": "^13.0.6"
|
|
69
|
+
"glob": "^13.0.6",
|
|
70
|
+
"js-yaml": "^4.1.1"
|
|
70
71
|
}
|
|
71
72
|
}
|