@mdream/crawl 2.0.0-beta.1 → 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +194 -88
- package/dist/_chunks/crawl.mjs +237 -29
- package/dist/cli.mjs +328 -358
- package/dist/index.d.mts +64 -59
- package/package.json +14 -4
package/dist/cli.mjs
CHANGED
|
@@ -1,10 +1,202 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { CRAWL_ARTIFACTS, CRAWL_DEFAULTS, assertCrawlOptionKeys, mergeCrawlOptions, resolveCrawlOptions, resolveLogger, runCrawl, validateCrawlUrl } from "./_chunks/crawl.mjs";
|
|
2
2
|
import { accessSync, constants, mkdirSync, readFileSync, unlinkSync, writeFileSync } from "node:fs";
|
|
3
|
-
import { dirname, join, relative
|
|
4
|
-
import { withHttps } from "ufo";
|
|
3
|
+
import { dirname, join, relative } from "pathe";
|
|
5
4
|
import * as p from "@clack/prompts";
|
|
6
5
|
import { fileURLToPath } from "node:url";
|
|
7
6
|
import { loadConfig } from "c12";
|
|
7
|
+
function toList(value) {
|
|
8
|
+
if (value === void 0) return [];
|
|
9
|
+
return typeof value === "string" ? [value] : value;
|
|
10
|
+
}
|
|
11
|
+
function addUrl(options, url) {
|
|
12
|
+
options.urls = [...options.urls ?? [], url];
|
|
13
|
+
}
|
|
14
|
+
const FLAGS = {
|
|
15
|
+
"--url": {
|
|
16
|
+
_tag: "Text",
|
|
17
|
+
apply: addUrl
|
|
18
|
+
},
|
|
19
|
+
"--output": {
|
|
20
|
+
_tag: "Text",
|
|
21
|
+
apply: (o, v) => {
|
|
22
|
+
o.output = v;
|
|
23
|
+
}
|
|
24
|
+
},
|
|
25
|
+
"--depth": {
|
|
26
|
+
_tag: "Number",
|
|
27
|
+
apply: (o, v) => {
|
|
28
|
+
o.depth = v;
|
|
29
|
+
}
|
|
30
|
+
},
|
|
31
|
+
"--single-page": {
|
|
32
|
+
_tag: "Switch",
|
|
33
|
+
apply: () => {}
|
|
34
|
+
},
|
|
35
|
+
"--driver": {
|
|
36
|
+
_tag: "Text",
|
|
37
|
+
apply: (o, v) => {
|
|
38
|
+
o.driver = v;
|
|
39
|
+
}
|
|
40
|
+
},
|
|
41
|
+
"--artifacts": {
|
|
42
|
+
_tag: "Text",
|
|
43
|
+
apply: (o, v) => {
|
|
44
|
+
o.artifacts = v.split(",").map((a) => a.trim());
|
|
45
|
+
}
|
|
46
|
+
},
|
|
47
|
+
"--origin": {
|
|
48
|
+
_tag: "Text",
|
|
49
|
+
apply: (o, v) => {
|
|
50
|
+
o.origin = v;
|
|
51
|
+
}
|
|
52
|
+
},
|
|
53
|
+
"--site-name": {
|
|
54
|
+
_tag: "Text",
|
|
55
|
+
apply: (o, v) => {
|
|
56
|
+
o.siteName = v;
|
|
57
|
+
}
|
|
58
|
+
},
|
|
59
|
+
"--description": {
|
|
60
|
+
_tag: "Text",
|
|
61
|
+
apply: (o, v) => {
|
|
62
|
+
o.description = v;
|
|
63
|
+
}
|
|
64
|
+
},
|
|
65
|
+
"--max-pages": {
|
|
66
|
+
_tag: "Number",
|
|
67
|
+
apply: (o, v) => {
|
|
68
|
+
o.maxPages = v;
|
|
69
|
+
}
|
|
70
|
+
},
|
|
71
|
+
"--crawl-delay": {
|
|
72
|
+
_tag: "Number",
|
|
73
|
+
apply: (o, v) => {
|
|
74
|
+
o.crawlDelay = v;
|
|
75
|
+
}
|
|
76
|
+
},
|
|
77
|
+
"--exclude": {
|
|
78
|
+
_tag: "Text",
|
|
79
|
+
apply: (o, v) => {
|
|
80
|
+
o.exclude = [...o.exclude ?? [], v];
|
|
81
|
+
}
|
|
82
|
+
},
|
|
83
|
+
"--skip-sitemap": {
|
|
84
|
+
_tag: "Switch",
|
|
85
|
+
apply: (o) => {
|
|
86
|
+
o.skipSitemap = true;
|
|
87
|
+
}
|
|
88
|
+
},
|
|
89
|
+
"--sitemap": {
|
|
90
|
+
_tag: "Text",
|
|
91
|
+
apply: (o, v) => {
|
|
92
|
+
o.sitemap = [...toList(o.sitemap), v];
|
|
93
|
+
}
|
|
94
|
+
},
|
|
95
|
+
"--allow-subdomains": {
|
|
96
|
+
_tag: "Switch",
|
|
97
|
+
apply: (o) => {
|
|
98
|
+
o.allowSubdomains = true;
|
|
99
|
+
}
|
|
100
|
+
},
|
|
101
|
+
"--keep-boilerplate": {
|
|
102
|
+
_tag: "Switch",
|
|
103
|
+
apply: (o) => {
|
|
104
|
+
o.stripBoilerplate = false;
|
|
105
|
+
}
|
|
106
|
+
},
|
|
107
|
+
"--boilerplate-threshold": {
|
|
108
|
+
_tag: "Number",
|
|
109
|
+
apply: (o, v) => {
|
|
110
|
+
o.boilerplateThreshold = v;
|
|
111
|
+
}
|
|
112
|
+
},
|
|
113
|
+
"--verbose": {
|
|
114
|
+
_tag: "Switch",
|
|
115
|
+
apply: (o) => {
|
|
116
|
+
o.verbose = true;
|
|
117
|
+
}
|
|
118
|
+
},
|
|
119
|
+
"--quiet": {
|
|
120
|
+
_tag: "Switch",
|
|
121
|
+
apply: (o) => {
|
|
122
|
+
o.silent = true;
|
|
123
|
+
}
|
|
124
|
+
},
|
|
125
|
+
"--silent": {
|
|
126
|
+
_tag: "Switch",
|
|
127
|
+
apply: (o) => {
|
|
128
|
+
o.silent = true;
|
|
129
|
+
}
|
|
130
|
+
}
|
|
131
|
+
};
|
|
132
|
+
const SHORT_FLAGS = {
|
|
133
|
+
"-u": "--url",
|
|
134
|
+
"-o": "--output",
|
|
135
|
+
"-d": "--depth",
|
|
136
|
+
"-v": "--verbose",
|
|
137
|
+
"-q": "--quiet"
|
|
138
|
+
};
|
|
139
|
+
function invalid(message) {
|
|
140
|
+
return {
|
|
141
|
+
_tag: "Invalid",
|
|
142
|
+
message
|
|
143
|
+
};
|
|
144
|
+
}
|
|
145
|
+
function unknownFlag(flag) {
|
|
146
|
+
if (flag[1] !== "-" && Object.hasOwn(FLAGS, `-${flag}`)) return invalid(`Unknown flag "${flag}". Use "-${flag}".`);
|
|
147
|
+
return invalid(`Unknown flag "${flag}". Run "mdream-crawl --help" to list the flags.`);
|
|
148
|
+
}
|
|
149
|
+
function isFlag(arg) {
|
|
150
|
+
return arg.startsWith("--") || Object.hasOwn(SHORT_FLAGS, arg);
|
|
151
|
+
}
|
|
152
|
+
function parseCrawlArgs(argv) {
|
|
153
|
+
if (argv.length === 0) return { _tag: "Interactive" };
|
|
154
|
+
if (argv.includes("--help") || argv.includes("-h")) return { _tag: "Help" };
|
|
155
|
+
if (argv.includes("--version")) return { _tag: "Version" };
|
|
156
|
+
const options = {};
|
|
157
|
+
let singlePage = false;
|
|
158
|
+
let positionalOnly = false;
|
|
159
|
+
for (let i = 0; i < argv.length; i++) {
|
|
160
|
+
const arg = argv[i];
|
|
161
|
+
if (positionalOnly || arg[0] !== "-") {
|
|
162
|
+
addUrl(options, arg);
|
|
163
|
+
continue;
|
|
164
|
+
}
|
|
165
|
+
if (arg === "--") {
|
|
166
|
+
positionalOnly = true;
|
|
167
|
+
continue;
|
|
168
|
+
}
|
|
169
|
+
const equals = arg.startsWith("--") ? arg.indexOf("=") : -1;
|
|
170
|
+
const written = equals === -1 ? arg : arg.slice(0, equals);
|
|
171
|
+
const name = Object.hasOwn(SHORT_FLAGS, written) ? SHORT_FLAGS[written] : written;
|
|
172
|
+
if (!Object.hasOwn(FLAGS, name)) return unknownFlag(written);
|
|
173
|
+
const spec = FLAGS[name];
|
|
174
|
+
if (spec._tag === "Switch") {
|
|
175
|
+
if (equals !== -1) return invalid(`"${written}" takes no value.`);
|
|
176
|
+
if (name === "--single-page") singlePage = true;
|
|
177
|
+
spec.apply(options);
|
|
178
|
+
continue;
|
|
179
|
+
}
|
|
180
|
+
let value;
|
|
181
|
+
if (equals !== -1) value = arg.slice(equals + 1);
|
|
182
|
+
else {
|
|
183
|
+
const next = argv[i + 1];
|
|
184
|
+
if (next === void 0 || isFlag(next)) return invalid(`"${written}" needs a value.`);
|
|
185
|
+
value = next;
|
|
186
|
+
i++;
|
|
187
|
+
}
|
|
188
|
+
if (spec._tag === "Number") {
|
|
189
|
+
const number = Number(value);
|
|
190
|
+
if (value.trim() === "" || Number.isNaN(number)) return invalid(`"${written}" needs a number. You gave ${JSON.stringify(value)}.`);
|
|
191
|
+
spec.apply(options, number);
|
|
192
|
+
} else spec.apply(options, value);
|
|
193
|
+
}
|
|
194
|
+
if (singlePage) options.depth = 0;
|
|
195
|
+
return {
|
|
196
|
+
_tag: "Crawl",
|
|
197
|
+
options
|
|
198
|
+
};
|
|
199
|
+
}
|
|
8
200
|
async function loadMdreamConfig(cwd) {
|
|
9
201
|
const { config } = await loadConfig({
|
|
10
202
|
name: "mdream",
|
|
@@ -24,6 +216,53 @@ let runLogger = resolveLogger({ silent: isSilentArgv(process.argv.slice(2)) });
|
|
|
24
216
|
const __dirname = dirname(fileURLToPath(import.meta.url));
|
|
25
217
|
const packageJsonPath = join(__dirname, "..", "package.json");
|
|
26
218
|
const version = JSON.parse(readFileSync(packageJsonPath, "utf-8")).version;
|
|
219
|
+
const HELP = `
|
|
220
|
+
@mdream/crawl v${version}
|
|
221
|
+
|
|
222
|
+
Crawl a website and generate llms.txt, llms-full.txt, and Markdown files.
|
|
223
|
+
|
|
224
|
+
Usage:
|
|
225
|
+
mdream-crawl [options] <url> Crawl a website
|
|
226
|
+
mdream-crawl Start interactive mode
|
|
227
|
+
npx @mdream/crawl [options] <url>
|
|
228
|
+
|
|
229
|
+
Options:
|
|
230
|
+
-u, --url <url> URL to crawl. Accepts glob patterns. Repeatable
|
|
231
|
+
-o, --output <dir> Output directory (default: ${CRAWL_DEFAULTS.output})
|
|
232
|
+
-d, --depth <number> Link depth from 0 to 10. 0 processes only the given URLs (default: ${CRAWL_DEFAULTS.depth})
|
|
233
|
+
--single-page Same as --depth 0
|
|
234
|
+
--driver <http|playwright> Crawler driver (default: ${CRAWL_DEFAULTS.driver})
|
|
235
|
+
--artifacts <list> Comma-separated list of ${CRAWL_ARTIFACTS.join(", ")} (default: all)
|
|
236
|
+
--origin <url> Origin for relative links in page Markdown (default: the page origin)
|
|
237
|
+
--site-name <name> Site name in llms.txt (default: the home page title)
|
|
238
|
+
--description <text> Site description in llms.txt (default: the home page description)
|
|
239
|
+
--max-pages <number> Maximum number of pages to fetch (default: no limit)
|
|
240
|
+
--crawl-delay <seconds> Delay between requests (default: the robots.txt Crawl-delay)
|
|
241
|
+
--exclude <pattern> Skip URLs that match a glob pattern. Repeatable
|
|
242
|
+
--skip-sitemap Skip sitemap and robots.txt discovery
|
|
243
|
+
--sitemap <url> Use this sitemap instead of auto-discovery. Repeatable
|
|
244
|
+
--allow-subdomains Crawl other subdomains of the same domain
|
|
245
|
+
--keep-boilerplate Keep repeated navigation and footers in page Markdown
|
|
246
|
+
--boilerplate-threshold <n> Fraction of pages that a block must appear in to count as boilerplate (default: ${CRAWL_DEFAULTS.boilerplateThreshold})
|
|
247
|
+
-v, --verbose Log details for each failed URL
|
|
248
|
+
-q, --quiet, --silent Drop all logs. Use it to keep stdout clean for JSON-RPC or MCP
|
|
249
|
+
-h, --help Show this help
|
|
250
|
+
--version Show the version
|
|
251
|
+
|
|
252
|
+
A flag can also take its value as --flag=value.
|
|
253
|
+
Flags override mdream.config.* values. Config values override the defaults.
|
|
254
|
+
|
|
255
|
+
Examples:
|
|
256
|
+
mdream-crawl harlanzw.com --artifacts "llms.txt,markdown"
|
|
257
|
+
mdream-crawl https://docs.example.com --depth 2 --artifacts llms-full.txt
|
|
258
|
+
mdream-crawl example.com --exclude "*/admin/*" --exclude "*/api/*"
|
|
259
|
+
mdream-crawl example.com --sitemap https://example.com/custom/sitemap.xml
|
|
260
|
+
mdream-crawl example.com/pricing --driver playwright --single-page
|
|
261
|
+
`;
|
|
262
|
+
function exitWithError(message) {
|
|
263
|
+
console.error(`Error: ${message}`);
|
|
264
|
+
process.exit(1);
|
|
265
|
+
}
|
|
27
266
|
function checkOutputDirectoryPermissions(outputDir) {
|
|
28
267
|
try {
|
|
29
268
|
mkdirSync(outputDir, { recursive: true });
|
|
@@ -56,27 +295,22 @@ function checkOutputDirectoryPermissions(outputDir) {
|
|
|
56
295
|
};
|
|
57
296
|
}
|
|
58
297
|
}
|
|
59
|
-
|
|
298
|
+
function splitUrls(input) {
|
|
299
|
+
return input.split(",").map((url) => url.trim()).filter(Boolean);
|
|
300
|
+
}
|
|
301
|
+
async function interactiveCrawl(config) {
|
|
60
302
|
console.clear();
|
|
61
303
|
p.intro(`☁️ @mdream/crawl v${version}`);
|
|
62
304
|
const urlsInput = await p.text({
|
|
63
305
|
message: "Enter starting URL for crawling (supports glob patterns):",
|
|
64
306
|
placeholder: "e.g. docs.example.com, site.com/docs/**",
|
|
307
|
+
initialValue: config.urls?.join(", "),
|
|
65
308
|
validate: (value) => {
|
|
66
|
-
|
|
67
|
-
|
|
309
|
+
const urls = splitUrls(value ?? "");
|
|
310
|
+
if (urls.length === 0) return "Enter at least one URL.";
|
|
68
311
|
for (const url of urls) {
|
|
69
|
-
const
|
|
70
|
-
if (
|
|
71
|
-
try {
|
|
72
|
-
if (!parseUrlPattern(url).isGlob) try {
|
|
73
|
-
new URL(withHttps(url));
|
|
74
|
-
} catch {
|
|
75
|
-
return `Invalid URL: ${withHttps(url)}`;
|
|
76
|
-
}
|
|
77
|
-
} catch (error) {
|
|
78
|
-
return error instanceof Error ? error.message : "Invalid URL pattern";
|
|
79
|
-
}
|
|
312
|
+
const error = validateCrawlUrl(url);
|
|
313
|
+
if (error) return error;
|
|
80
314
|
}
|
|
81
315
|
}
|
|
82
316
|
});
|
|
@@ -84,107 +318,68 @@ async function interactiveCrawl() {
|
|
|
84
318
|
p.cancel("Operation cancelled.");
|
|
85
319
|
return null;
|
|
86
320
|
}
|
|
87
|
-
const
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
}, {
|
|
103
|
-
value: "playwright",
|
|
104
|
-
label: "Playwright (Slower, supports JavaScript)"
|
|
105
|
-
}],
|
|
106
|
-
initialValue: "http"
|
|
107
|
-
}) }, { onCancel: () => {
|
|
108
|
-
p.cancel("Operation cancelled.");
|
|
109
|
-
process.exit(0);
|
|
110
|
-
} });
|
|
111
|
-
const advancedOptions = await p.group({
|
|
112
|
-
outputFormats: () => p.multiselect({
|
|
113
|
-
message: "Select output formats:",
|
|
321
|
+
const answers = await p.group({
|
|
322
|
+
driver: () => p.select({
|
|
323
|
+
message: "Select crawler driver:",
|
|
324
|
+
options: [{
|
|
325
|
+
value: "http",
|
|
326
|
+
label: "HTTP Crawler (Fast, for static content)",
|
|
327
|
+
hint: "Recommended"
|
|
328
|
+
}, {
|
|
329
|
+
value: "playwright",
|
|
330
|
+
label: "Playwright (Slower, supports JavaScript)"
|
|
331
|
+
}],
|
|
332
|
+
initialValue: config.driver ?? CRAWL_DEFAULTS.driver
|
|
333
|
+
}),
|
|
334
|
+
artifacts: () => p.multiselect({
|
|
335
|
+
message: "Select artifacts:",
|
|
114
336
|
options: [
|
|
115
337
|
{
|
|
116
338
|
value: "llms.txt",
|
|
117
|
-
label: "llms.txt (
|
|
339
|
+
label: "llms.txt (link index)",
|
|
118
340
|
hint: "Recommended"
|
|
119
341
|
},
|
|
120
342
|
{
|
|
121
343
|
value: "llms-full.txt",
|
|
122
|
-
label: "llms-full.txt (
|
|
344
|
+
label: "llms-full.txt (full content)"
|
|
123
345
|
},
|
|
124
346
|
{
|
|
125
347
|
value: "markdown",
|
|
126
|
-
label: "
|
|
348
|
+
label: "Markdown file for each page"
|
|
127
349
|
}
|
|
128
350
|
],
|
|
129
|
-
initialValues: [
|
|
130
|
-
"llms.txt",
|
|
131
|
-
"llms-full.txt",
|
|
132
|
-
"markdown"
|
|
133
|
-
]
|
|
351
|
+
initialValues: config.artifacts ?? [...CRAWL_DEFAULTS.artifacts]
|
|
134
352
|
}),
|
|
135
353
|
skipSitemap: () => p.confirm({
|
|
136
354
|
message: "Skip sitemap.xml and robots.txt discovery?",
|
|
137
|
-
initialValue: false
|
|
355
|
+
initialValue: config.skipSitemap ?? false
|
|
138
356
|
})
|
|
139
357
|
}, { onCancel: () => {
|
|
140
358
|
p.cancel("Operation cancelled.");
|
|
141
359
|
process.exit(0);
|
|
142
360
|
} });
|
|
143
|
-
const firstUrl = urls[0];
|
|
144
|
-
const inferredOrigin = (() => {
|
|
145
|
-
try {
|
|
146
|
-
const url = new URL(withHttps(firstUrl));
|
|
147
|
-
return `${url.protocol}//${url.host}`;
|
|
148
|
-
} catch {
|
|
149
|
-
return;
|
|
150
|
-
}
|
|
151
|
-
})();
|
|
152
|
-
const outputFormats = advancedOptions.outputFormats.map((f) => {
|
|
153
|
-
switch (f) {
|
|
154
|
-
case "llms.txt": return "llms.txt";
|
|
155
|
-
case "llms-full.txt": return "llms-full.txt";
|
|
156
|
-
case "markdown": return "Individual MD files";
|
|
157
|
-
default: return f;
|
|
158
|
-
}
|
|
159
|
-
});
|
|
160
|
-
const summary = [
|
|
161
|
-
`URLs: ${urls.join(", ")}`,
|
|
162
|
-
`Output: ${relative(process.cwd(), outputDir) || outputDir}`,
|
|
163
|
-
`Driver: ${crawlerOptions.driver}`,
|
|
164
|
-
`Max pages: Unlimited`,
|
|
165
|
-
`Follow links: Yes (depth 3)`,
|
|
166
|
-
`Output formats: ${outputFormats.join(", ")}`,
|
|
167
|
-
`Sitemap discovery: ${advancedOptions.skipSitemap ? "Skipped" : "Automatic"}`,
|
|
168
|
-
inferredOrigin && `Origin: ${inferredOrigin}`
|
|
169
|
-
].filter(Boolean);
|
|
170
|
-
p.note(summary.join("\n"), "Crawl Configuration");
|
|
171
|
-
if (advancedOptions.skipSitemap && globPatterns.some((p) => p.isGlob)) p.log.warn("Warning: Using --skip-sitemap with glob URLs may not discover all matching pages.");
|
|
172
361
|
return {
|
|
173
|
-
urls,
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
followLinks: true,
|
|
178
|
-
generateLlmsTxt: advancedOptions.outputFormats.includes("llms.txt"),
|
|
179
|
-
generateLlmsFullTxt: advancedOptions.outputFormats.includes("llms-full.txt"),
|
|
180
|
-
generateIndividualMd: advancedOptions.outputFormats.includes("markdown"),
|
|
181
|
-
origin: inferredOrigin,
|
|
182
|
-
globPatterns,
|
|
183
|
-
verbose: false,
|
|
184
|
-
maxDepth: 3,
|
|
185
|
-
skipSitemap: advancedOptions.skipSitemap
|
|
362
|
+
urls: splitUrls(urlsInput),
|
|
363
|
+
driver: answers.driver,
|
|
364
|
+
artifacts: answers.artifacts,
|
|
365
|
+
skipSitemap: answers.skipSitemap
|
|
186
366
|
};
|
|
187
367
|
}
|
|
368
|
+
function summarize(options) {
|
|
369
|
+
const maxPages = Number.isFinite(options.maxPages) ? String(options.maxPages) : "no limit";
|
|
370
|
+
return [
|
|
371
|
+
`URL: ${options.urls.join(", ")}`,
|
|
372
|
+
`Output: ${relative(process.cwd(), options.output) || "."}`,
|
|
373
|
+
`Driver: ${options.driver} · Depth: ${options.depth} · Max pages: ${maxPages}`,
|
|
374
|
+
`Artifacts: ${options.artifacts.length > 0 ? options.artifacts.join(", ") : "none"}`,
|
|
375
|
+
options.origin && `Origin: ${options.origin}`,
|
|
376
|
+
options.exclude.length > 0 && `Exclude: ${options.exclude.join(", ")}`,
|
|
377
|
+
options.skipSitemap && `Skip sitemap: yes`,
|
|
378
|
+
options.sitemap.length > 0 && `Sitemap: ${options.sitemap.join(", ")}`,
|
|
379
|
+
options.allowSubdomains && `Allow subdomains: yes`,
|
|
380
|
+
options.verbose && `Verbose: on`
|
|
381
|
+
].filter(Boolean).join("\n");
|
|
382
|
+
}
|
|
188
383
|
async function showCrawlResults(logger, successful, failed, outputDir, generatedFiles, durationSeconds, latency) {
|
|
189
384
|
let line = `${successful} pages in ${`${durationSeconds.toFixed(1)}s`}`;
|
|
190
385
|
if (failed > 0) line += ` (${failed} failed)`;
|
|
@@ -192,261 +387,45 @@ async function showCrawlResults(logger, successful, failed, outputDir, generated
|
|
|
192
387
|
const avg = Math.round(latency.total / latency.count);
|
|
193
388
|
const min = latency.min === Infinity ? 0 : Math.round(latency.min);
|
|
194
389
|
const max = Math.round(latency.max);
|
|
195
|
-
line += `
|
|
390
|
+
line += ` · HTTP Latency: avg ${avg}ms, min ${min}ms, max ${max}ms`;
|
|
196
391
|
}
|
|
197
392
|
logger.success(line);
|
|
198
|
-
logger.info(`${generatedFiles.join(", ")}
|
|
393
|
+
if (generatedFiles.length > 0) logger.info(`${generatedFiles.join(", ")} → ${relative(process.cwd(), outputDir) || "."}`);
|
|
199
394
|
}
|
|
200
|
-
function
|
|
201
|
-
const
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
if (args.includes("--help") || args.includes("-h")) {
|
|
205
|
-
console.log(`
|
|
206
|
-
@mdream/crawl v${version}
|
|
207
|
-
|
|
208
|
-
Multi-page website crawler that generates comprehensive llms.txt files
|
|
209
|
-
|
|
210
|
-
Usage:
|
|
211
|
-
@mdream/crawl [options] <url> Crawl a website with CLI flags
|
|
212
|
-
@mdream/crawl Start interactive mode
|
|
213
|
-
|
|
214
|
-
Options:
|
|
215
|
-
-u, --url <url> Website URL to crawl
|
|
216
|
-
-o, --output <dir> Output directory (default: output)
|
|
217
|
-
-d, --depth <number> Crawl depth, 0 for single page (default: 3)
|
|
218
|
-
--single-page Only process the given URL(s), no crawling (alias for --depth 0)
|
|
219
|
-
--driver <http|playwright> Crawler driver (default: http)
|
|
220
|
-
--artifacts <list> Comma-separated list of artifacts: llms.txt,llms-full.txt,markdown (default: all)
|
|
221
|
-
--origin <url> Origin URL for resolving relative paths (overrides auto-detection)
|
|
222
|
-
--site-name <name> Override site name (overrides auto-extracted title)
|
|
223
|
-
--description <desc> Override site description (overrides auto-extracted description)
|
|
224
|
-
--max-pages <number> Maximum pages to crawl (default: unlimited)
|
|
225
|
-
--crawl-delay <seconds> Crawl delay in seconds
|
|
226
|
-
--exclude <pattern> Exclude URLs matching glob patterns (can be used multiple times)
|
|
227
|
-
--skip-sitemap Skip sitemap.xml and robots.txt discovery
|
|
228
|
-
--sitemap <url> Use an explicit sitemap URL instead of auto-discovery (repeatable for multi-part sitemaps)
|
|
229
|
-
--allow-subdomains Crawl across subdomains of the same root domain
|
|
230
|
-
--keep-boilerplate Keep repeated site chrome (nav/footer) in per-page output (default: stripped)
|
|
231
|
-
--boilerplate-threshold <n> Fraction of pages a block must repeat in to count as chrome (0-1, default: ${DEFAULT_BOILERPLATE_THRESHOLD})
|
|
232
|
-
-v, --verbose Enable verbose logging
|
|
233
|
-
-q, --quiet, --silent Suppress all logs (clean stdout for JSON-RPC/MCP)
|
|
234
|
-
-h, --help Show this help message
|
|
235
|
-
--version Show version number
|
|
236
|
-
|
|
237
|
-
Note: Sitemap discovery and robots.txt checking are automatic unless --skip-sitemap is used.
|
|
238
|
-
|
|
239
|
-
Examples:
|
|
240
|
-
@mdream/crawl -u harlanzw.com --artifacts "llms.txt,markdown"
|
|
241
|
-
@mdream/crawl --url https://docs.example.com --depth 2 --artifacts "llms-full.txt"
|
|
242
|
-
@mdream/crawl -u example.com --exclude "*/admin/*" --exclude "*/api/*"
|
|
243
|
-
@mdream/crawl -u example.com --verbose
|
|
244
|
-
@mdream/crawl -u example.com --skip-sitemap
|
|
245
|
-
@mdream/crawl -u example.com --sitemap https://example.com/custom/sitemap.xml
|
|
246
|
-
@mdream/crawl -u example.com --driver playwright --single-page
|
|
247
|
-
`);
|
|
395
|
+
async function main() {
|
|
396
|
+
const command = parseCrawlArgs(process.argv.slice(2));
|
|
397
|
+
if (command._tag === "Help") {
|
|
398
|
+
console.log(HELP);
|
|
248
399
|
process.exit(0);
|
|
249
400
|
}
|
|
250
|
-
if (
|
|
401
|
+
if (command._tag === "Version") {
|
|
251
402
|
console.log(`@mdream/crawl v${version}`);
|
|
252
403
|
process.exit(0);
|
|
253
404
|
}
|
|
254
|
-
if (
|
|
255
|
-
const
|
|
256
|
-
const index = args.findIndex((arg) => arg === flag || arg === flag.replace("--", "-"));
|
|
257
|
-
return index >= 0 && index + 1 < args.length ? args[index + 1] : void 0;
|
|
258
|
-
};
|
|
259
|
-
const getArgValues = (flag) => {
|
|
260
|
-
const values = [];
|
|
261
|
-
for (let i = 0; i < args.length; i++) if (args[i] === flag || args[i] === flag.replace("--", "-")) {
|
|
262
|
-
if (i + 1 < args.length && !args[i + 1].startsWith("-")) values.push(args[i + 1]);
|
|
263
|
-
}
|
|
264
|
-
return values;
|
|
265
|
-
};
|
|
266
|
-
const urlFromFlag = getArgValue("--url") || getArgValue("-u");
|
|
267
|
-
const urlFromArgs = args.find((arg) => !arg.startsWith("-") && !args[args.indexOf(arg) - 1]?.startsWith("-"));
|
|
268
|
-
const url = urlFromFlag || urlFromArgs;
|
|
269
|
-
if (!url) {
|
|
270
|
-
logger.error("Error: URL is required when using CLI arguments");
|
|
271
|
-
logger.info("Use --help for usage information or run without arguments for interactive mode");
|
|
272
|
-
process.exit(1);
|
|
273
|
-
}
|
|
274
|
-
const globError = validateGlobPattern(url);
|
|
275
|
-
if (globError) {
|
|
276
|
-
logger.error(`Error: ${globError}`);
|
|
277
|
-
process.exit(1);
|
|
278
|
-
}
|
|
279
|
-
let parsed;
|
|
405
|
+
if (command._tag === "Invalid") exitWithError(command.message);
|
|
406
|
+
const fileConfig = await loadMdreamConfig();
|
|
280
407
|
try {
|
|
281
|
-
|
|
408
|
+
assertCrawlOptionKeys(fileConfig);
|
|
282
409
|
} catch (error) {
|
|
283
|
-
|
|
284
|
-
process.exit(1);
|
|
285
|
-
}
|
|
286
|
-
if (!parsed.isGlob) try {
|
|
287
|
-
new URL(withHttps(url));
|
|
288
|
-
} catch {
|
|
289
|
-
logger.error(`Error: Invalid URL: ${withHttps(url)}`);
|
|
290
|
-
process.exit(1);
|
|
291
|
-
}
|
|
292
|
-
const excludePatterns = getArgValues("--exclude");
|
|
293
|
-
for (const pattern of excludePatterns) {
|
|
294
|
-
const excludeError = validateGlobPattern(pattern);
|
|
295
|
-
if (excludeError) {
|
|
296
|
-
logger.error(`Error in exclude pattern: ${excludeError}`);
|
|
297
|
-
process.exit(1);
|
|
298
|
-
}
|
|
299
|
-
}
|
|
300
|
-
const depthStr = args.includes("--single-page") ? "0" : getArgValue("--depth") || getArgValue("-d") || "3";
|
|
301
|
-
const depth = Number(depthStr);
|
|
302
|
-
if (!Number.isInteger(depth) || depth < 0 || depth > 10) {
|
|
303
|
-
logger.error("Error: Depth must be an integer between 0 and 10");
|
|
304
|
-
process.exit(1);
|
|
305
|
-
}
|
|
306
|
-
const driver = getArgValue("--driver");
|
|
307
|
-
if (driver && driver !== "http" && driver !== "playwright") {
|
|
308
|
-
logger.error("Error: Driver must be either \"http\" or \"playwright\"");
|
|
309
|
-
process.exit(1);
|
|
410
|
+
exitWithError(`mdream.config: ${error instanceof Error ? error.message : String(error)}`);
|
|
310
411
|
}
|
|
311
|
-
const
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
const crawlDelay = Number.parseInt(crawlDelayStr);
|
|
322
|
-
if (Number.isNaN(crawlDelay) || crawlDelay < 0) {
|
|
323
|
-
logger.error("Error: Crawl delay must be a non-negative number");
|
|
324
|
-
process.exit(1);
|
|
325
|
-
}
|
|
326
|
-
}
|
|
327
|
-
const artifactsStr = getArgValue("--artifacts");
|
|
328
|
-
const artifacts = artifactsStr ? artifactsStr.split(",").map((a) => a.trim()) : [
|
|
329
|
-
"llms.txt",
|
|
330
|
-
"llms-full.txt",
|
|
331
|
-
"markdown"
|
|
332
|
-
];
|
|
333
|
-
const validArtifacts = [
|
|
334
|
-
"llms.txt",
|
|
335
|
-
"llms-full.txt",
|
|
336
|
-
"markdown"
|
|
337
|
-
];
|
|
338
|
-
for (const artifact of artifacts) if (!validArtifacts.includes(artifact)) {
|
|
339
|
-
logger.error(`Error: Invalid artifact '${artifact}'. Valid options: ${validArtifacts.join(", ")}`);
|
|
340
|
-
process.exit(1);
|
|
341
|
-
}
|
|
342
|
-
const originOverride = getArgValue("--origin");
|
|
343
|
-
const inferredOrigin = (() => {
|
|
344
|
-
if (originOverride) return originOverride;
|
|
345
|
-
try {
|
|
346
|
-
const urlObj = new URL(withHttps(url));
|
|
347
|
-
return `${urlObj.protocol}//${urlObj.host}`;
|
|
348
|
-
} catch {
|
|
349
|
-
return;
|
|
350
|
-
}
|
|
351
|
-
})();
|
|
352
|
-
const siteNameOverride = getArgValue("--site-name");
|
|
353
|
-
const descriptionOverride = getArgValue("--description");
|
|
354
|
-
const patterns = [parsed];
|
|
355
|
-
const verbose = args.includes("--verbose") || args.includes("-v");
|
|
356
|
-
const skipSitemap = args.includes("--skip-sitemap");
|
|
357
|
-
const sitemapUrls = getArgValues("--sitemap").map((u) => withHttps(u));
|
|
358
|
-
if (args.filter((a) => a === "--sitemap" || a === "-sitemap").length > sitemapUrls.length) {
|
|
359
|
-
logger.error("Error: --sitemap requires a URL value");
|
|
360
|
-
process.exit(1);
|
|
361
|
-
}
|
|
362
|
-
for (const sitemapUrl of sitemapUrls) try {
|
|
363
|
-
new URL(sitemapUrl);
|
|
364
|
-
} catch {
|
|
365
|
-
logger.error(`Error: Invalid sitemap URL: ${sitemapUrl}`);
|
|
366
|
-
process.exit(1);
|
|
367
|
-
}
|
|
368
|
-
const allowSubdomains = args.includes("--allow-subdomains");
|
|
369
|
-
const stripBoilerplate = args.includes("--keep-boilerplate") ? false : void 0;
|
|
370
|
-
const boilerplateThresholdStr = getArgValue("--boilerplate-threshold");
|
|
371
|
-
let boilerplateThreshold;
|
|
372
|
-
if (boilerplateThresholdStr !== void 0) {
|
|
373
|
-
boilerplateThreshold = Number(boilerplateThresholdStr);
|
|
374
|
-
if (Number.isNaN(boilerplateThreshold) || boilerplateThreshold <= 0 || boilerplateThreshold > 1) {
|
|
375
|
-
logger.error("Error: Boilerplate threshold must be a number between 0 (exclusive) and 1");
|
|
376
|
-
process.exit(1);
|
|
377
|
-
}
|
|
412
|
+
const interactive = command._tag === "Interactive";
|
|
413
|
+
const layer = interactive ? await interactiveCrawl(fileConfig) : command.options;
|
|
414
|
+
if (!layer) process.exit(0);
|
|
415
|
+
const input = mergeCrawlOptions(fileConfig, layer);
|
|
416
|
+
if (!input.urls || input.urls.length === 0) exitWithError("A URL is required. Pass it as an argument, with --url, or as \"urls\" in mdream.config. Run without arguments for interactive mode.");
|
|
417
|
+
let options;
|
|
418
|
+
try {
|
|
419
|
+
options = resolveCrawlOptions(input);
|
|
420
|
+
} catch (error) {
|
|
421
|
+
exitWithError(error instanceof Error ? error.message : String(error));
|
|
378
422
|
}
|
|
379
|
-
|
|
380
|
-
return {
|
|
381
|
-
urls: [url],
|
|
382
|
-
outputDir: resolve(getArgValue("--output") || getArgValue("-o") || "output"),
|
|
383
|
-
driver: driver || "http",
|
|
384
|
-
maxRequestsPerCrawl: Number.parseInt(maxPagesStr || String(Number.MAX_SAFE_INTEGER)),
|
|
385
|
-
followLinks: depth > 0,
|
|
386
|
-
maxDepth: depth,
|
|
387
|
-
generateLlmsTxt: artifacts.includes("llms.txt"),
|
|
388
|
-
generateLlmsFullTxt: artifacts.includes("llms-full.txt"),
|
|
389
|
-
generateIndividualMd: artifacts.includes("markdown"),
|
|
390
|
-
siteNameOverride,
|
|
391
|
-
descriptionOverride,
|
|
392
|
-
origin: inferredOrigin,
|
|
393
|
-
globPatterns: patterns,
|
|
394
|
-
crawlDelay: crawlDelayStr ? Number.parseInt(crawlDelayStr) : void 0,
|
|
395
|
-
exclude: excludePatterns.length > 0 ? excludePatterns : void 0,
|
|
396
|
-
verbose,
|
|
397
|
-
skipSitemap,
|
|
398
|
-
sitemapUrls: sitemapUrls.length > 0 ? sitemapUrls : void 0,
|
|
399
|
-
allowSubdomains,
|
|
400
|
-
stripBoilerplate,
|
|
401
|
-
boilerplateThreshold,
|
|
402
|
-
silent
|
|
403
|
-
};
|
|
404
|
-
}
|
|
405
|
-
async function main() {
|
|
406
|
-
const cliOptions = parseCliArgs();
|
|
407
|
-
const fileConfig = await loadMdreamConfig();
|
|
408
|
-
const silent = (cliOptions?.silent ?? false) || (fileConfig.silent ?? false);
|
|
409
|
-
const logger = resolveLogger({ silent });
|
|
423
|
+
const logger = options.logger;
|
|
410
424
|
runLogger = logger;
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
options = {
|
|
416
|
-
...cliOptions,
|
|
417
|
-
driver: cliOptions.driver || fileConfig.driver || "http",
|
|
418
|
-
maxDepth: cliOptions.maxDepth ?? fileConfig.maxDepth,
|
|
419
|
-
crawlDelay: cliOptions.crawlDelay ?? fileConfig.crawlDelay,
|
|
420
|
-
skipSitemap: cliOptions.skipSitemap || fileConfig.skipSitemap || false,
|
|
421
|
-
sitemapUrls: cliOptions.sitemapUrls ?? (fileConfig.sitemap ? Array.isArray(fileConfig.sitemap) ? fileConfig.sitemap : [fileConfig.sitemap] : void 0),
|
|
422
|
-
allowSubdomains: cliOptions.allowSubdomains || fileConfig.allowSubdomains || false,
|
|
423
|
-
stripBoilerplate: cliOptions.stripBoilerplate ?? fileConfig.stripBoilerplate ?? true,
|
|
424
|
-
boilerplateThreshold: cliOptions.boilerplateThreshold ?? fileConfig.boilerplateThreshold,
|
|
425
|
-
verbose: cliOptions.verbose || fileConfig.verbose || false,
|
|
426
|
-
silent,
|
|
427
|
-
exclude: configExclude.length > 0 || cliExclude.length > 0 ? [...configExclude, ...cliExclude] : void 0,
|
|
428
|
-
hooks: fileConfig.hooks
|
|
429
|
-
};
|
|
430
|
-
logger.intro(`☁️ mdream v${version}`);
|
|
431
|
-
const formats = [];
|
|
432
|
-
if (options.generateLlmsTxt) formats.push("llms.txt");
|
|
433
|
-
if (options.generateLlmsFullTxt) formats.push("llms-full.txt");
|
|
434
|
-
if (options.generateIndividualMd) formats.push("Individual MD files");
|
|
435
|
-
const summary = [
|
|
436
|
-
`URL: ${options.urls.join(", ")}`,
|
|
437
|
-
`Output: ${relative(process.cwd(), options.outputDir) || "."}`,
|
|
438
|
-
`Driver: ${options.driver} \u00B7 Depth: ${options.maxDepth}`,
|
|
439
|
-
`Formats: ${formats.join(", ")}`,
|
|
440
|
-
options.exclude && options.exclude.length > 0 && `Exclude: ${options.exclude.join(", ")}`,
|
|
441
|
-
options.skipSitemap && `Skip sitemap: Yes`,
|
|
442
|
-
options.sitemapUrls && options.sitemapUrls.length > 0 && `Sitemap: ${options.sitemapUrls.join(", ")}`,
|
|
443
|
-
options.allowSubdomains && `Allow subdomains: Yes`,
|
|
444
|
-
options.verbose && `Verbose: Enabled`
|
|
445
|
-
].filter(Boolean);
|
|
446
|
-
logger.note(summary.join("\n"), "Configuration");
|
|
447
|
-
} else options = await interactiveCrawl();
|
|
448
|
-
if (!options) process.exit(0);
|
|
449
|
-
const permCheck = checkOutputDirectoryPermissions(options.outputDir);
|
|
425
|
+
if (!interactive) logger.intro(`☁️ mdream v${version}`);
|
|
426
|
+
logger.note(summarize(options), "Configuration");
|
|
427
|
+
if (options.skipSitemap && options.patterns.some((pattern) => pattern.isGlob)) logger.warn("Sitemap discovery is off. A glob URL can miss pages that no link reaches.");
|
|
428
|
+
const permCheck = checkOutputDirectoryPermissions(options.output);
|
|
450
429
|
if (!permCheck.success) {
|
|
451
430
|
logger.error(permCheck.error);
|
|
452
431
|
if (permCheck.error?.includes("Permission denied")) logger.info("Tip: Try running with elevated privileges (e.g., sudo) or change the output directory permissions.");
|
|
@@ -460,8 +439,11 @@ async function main() {
|
|
|
460
439
|
process.exit(1);
|
|
461
440
|
}
|
|
462
441
|
const { ensurePlaywrightInstalled, isUseChromeSupported } = await import("./_chunks/playwright-utils.mjs");
|
|
463
|
-
if (await isUseChromeSupported()) {
|
|
464
|
-
options
|
|
442
|
+
if (input.useChrome !== false && await isUseChromeSupported()) {
|
|
443
|
+
options = {
|
|
444
|
+
...options,
|
|
445
|
+
useChrome: true
|
|
446
|
+
};
|
|
465
447
|
logger.info("System Chrome detected and enabled.");
|
|
466
448
|
} else {
|
|
467
449
|
if (!await ensurePlaywrightInstalled(logger)) {
|
|
@@ -476,7 +458,7 @@ async function main() {
|
|
|
476
458
|
const startTime = Date.now();
|
|
477
459
|
let crawlStartTime = 0;
|
|
478
460
|
let lastProgress;
|
|
479
|
-
const results = await
|
|
461
|
+
const results = await runCrawl(options, (progress) => {
|
|
480
462
|
lastProgress = progress;
|
|
481
463
|
if (progress.sitemap.status === "discovering") s.message("Discovering sitemaps");
|
|
482
464
|
else if (progress.sitemap.status === "processing") s.message(`Processing sitemap... Found ${progress.sitemap.found} URLs`);
|
|
@@ -488,34 +470,22 @@ async function main() {
|
|
|
488
470
|
const elapsed = (Date.now() - crawlStartTime) / 1e3;
|
|
489
471
|
const rate = elapsed > .1 ? Math.round(processed / elapsed) : 0;
|
|
490
472
|
let msg = processed > total ? `Crawling ${processed} pages` : `Crawling ${processed}/${total}`;
|
|
491
|
-
if (rate > 0) msg += `
|
|
492
|
-
if (failed > 0) msg += `
|
|
473
|
+
if (rate > 0) msg += ` · ${rate}/s`;
|
|
474
|
+
if (failed > 0) msg += ` · ${failed} failed`;
|
|
493
475
|
s.message(msg);
|
|
494
476
|
} else if (progress.generation.status === "generating") s.message(progress.generation.current || "Generating files");
|
|
495
477
|
});
|
|
496
478
|
s.stop();
|
|
497
479
|
const durationSeconds = (Date.now() - startTime) / 1e3;
|
|
498
480
|
const successful = results.filter((r) => r.success).length;
|
|
499
|
-
const failed = results.filter((r) => !r.success).length;
|
|
500
481
|
const failedResults = results.filter((r) => !r.success);
|
|
501
|
-
if (
|
|
482
|
+
if (failedResults.length > 0) {
|
|
502
483
|
logger.error("Failed URLs:");
|
|
503
|
-
failedResults.
|
|
504
|
-
logger.error(` ${result.url}: ${result.error || "Unknown error"}`);
|
|
505
|
-
});
|
|
506
|
-
} else if (failed > 0) {
|
|
507
|
-
console.log("\nFailed URLs:");
|
|
508
|
-
failedResults.forEach((result) => {
|
|
509
|
-
console.log(` - ${result.url}: ${result.error || "Unknown error"}`);
|
|
510
|
-
});
|
|
484
|
+
for (const result of failedResults) logger.error(` ${result.url}: ${result.error || "Unknown error"}`);
|
|
511
485
|
}
|
|
512
486
|
const generatedFiles = [];
|
|
513
|
-
if (successful > 0) {
|
|
514
|
-
|
|
515
|
-
if (options.generateLlmsFullTxt) generatedFiles.push("llms-full.txt");
|
|
516
|
-
if (options.generateIndividualMd) generatedFiles.push(`${successful} MD files`);
|
|
517
|
-
}
|
|
518
|
-
await showCrawlResults(logger, successful, failed, options.outputDir, generatedFiles, durationSeconds, lastProgress?.crawling.latency);
|
|
487
|
+
if (successful > 0) for (const artifact of options.artifacts) generatedFiles.push(artifact === "markdown" ? `${successful} MD files` : artifact);
|
|
488
|
+
await showCrawlResults(logger, successful, failedResults.length, options.output, generatedFiles, durationSeconds, lastProgress?.crawling.latency);
|
|
519
489
|
process.exit(0);
|
|
520
490
|
}
|
|
521
491
|
main().catch((error) => {
|