@mdream/crawl 1.7.2 → 2.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/cli.mjs CHANGED
@@ -1,10 +1,202 @@
1
- import { DEFAULT_BOILERPLATE_THRESHOLD, crawlAndGenerate, parseUrlPattern, resolveLogger, validateGlobPattern } from "./_chunks/crawl.mjs";
1
+ import { CRAWL_ARTIFACTS, CRAWL_DEFAULTS, assertCrawlOptionKeys, mergeCrawlOptions, resolveCrawlOptions, resolveLogger, runCrawl, validateCrawlUrl } from "./_chunks/crawl.mjs";
2
2
  import { accessSync, constants, mkdirSync, readFileSync, unlinkSync, writeFileSync } from "node:fs";
3
- import { dirname, join, relative, resolve } from "pathe";
4
- import { withHttps } from "ufo";
3
+ import { dirname, join, relative } from "pathe";
5
4
  import * as p from "@clack/prompts";
6
5
  import { fileURLToPath } from "node:url";
7
6
  import { loadConfig } from "c12";
7
+ function toList(value) {
8
+ if (value === void 0) return [];
9
+ return typeof value === "string" ? [value] : value;
10
+ }
11
+ function addUrl(options, url) {
12
+ options.urls = [...options.urls ?? [], url];
13
+ }
14
+ const FLAGS = {
15
+ "--url": {
16
+ _tag: "Text",
17
+ apply: addUrl
18
+ },
19
+ "--output": {
20
+ _tag: "Text",
21
+ apply: (o, v) => {
22
+ o.output = v;
23
+ }
24
+ },
25
+ "--depth": {
26
+ _tag: "Number",
27
+ apply: (o, v) => {
28
+ o.depth = v;
29
+ }
30
+ },
31
+ "--single-page": {
32
+ _tag: "Switch",
33
+ apply: () => {}
34
+ },
35
+ "--driver": {
36
+ _tag: "Text",
37
+ apply: (o, v) => {
38
+ o.driver = v;
39
+ }
40
+ },
41
+ "--artifacts": {
42
+ _tag: "Text",
43
+ apply: (o, v) => {
44
+ o.artifacts = v.split(",").map((a) => a.trim());
45
+ }
46
+ },
47
+ "--origin": {
48
+ _tag: "Text",
49
+ apply: (o, v) => {
50
+ o.origin = v;
51
+ }
52
+ },
53
+ "--site-name": {
54
+ _tag: "Text",
55
+ apply: (o, v) => {
56
+ o.siteName = v;
57
+ }
58
+ },
59
+ "--description": {
60
+ _tag: "Text",
61
+ apply: (o, v) => {
62
+ o.description = v;
63
+ }
64
+ },
65
+ "--max-pages": {
66
+ _tag: "Number",
67
+ apply: (o, v) => {
68
+ o.maxPages = v;
69
+ }
70
+ },
71
+ "--crawl-delay": {
72
+ _tag: "Number",
73
+ apply: (o, v) => {
74
+ o.crawlDelay = v;
75
+ }
76
+ },
77
+ "--exclude": {
78
+ _tag: "Text",
79
+ apply: (o, v) => {
80
+ o.exclude = [...o.exclude ?? [], v];
81
+ }
82
+ },
83
+ "--skip-sitemap": {
84
+ _tag: "Switch",
85
+ apply: (o) => {
86
+ o.skipSitemap = true;
87
+ }
88
+ },
89
+ "--sitemap": {
90
+ _tag: "Text",
91
+ apply: (o, v) => {
92
+ o.sitemap = [...toList(o.sitemap), v];
93
+ }
94
+ },
95
+ "--allow-subdomains": {
96
+ _tag: "Switch",
97
+ apply: (o) => {
98
+ o.allowSubdomains = true;
99
+ }
100
+ },
101
+ "--keep-boilerplate": {
102
+ _tag: "Switch",
103
+ apply: (o) => {
104
+ o.stripBoilerplate = false;
105
+ }
106
+ },
107
+ "--boilerplate-threshold": {
108
+ _tag: "Number",
109
+ apply: (o, v) => {
110
+ o.boilerplateThreshold = v;
111
+ }
112
+ },
113
+ "--verbose": {
114
+ _tag: "Switch",
115
+ apply: (o) => {
116
+ o.verbose = true;
117
+ }
118
+ },
119
+ "--quiet": {
120
+ _tag: "Switch",
121
+ apply: (o) => {
122
+ o.silent = true;
123
+ }
124
+ },
125
+ "--silent": {
126
+ _tag: "Switch",
127
+ apply: (o) => {
128
+ o.silent = true;
129
+ }
130
+ }
131
+ };
132
+ const SHORT_FLAGS = {
133
+ "-u": "--url",
134
+ "-o": "--output",
135
+ "-d": "--depth",
136
+ "-v": "--verbose",
137
+ "-q": "--quiet"
138
+ };
139
+ function invalid(message) {
140
+ return {
141
+ _tag: "Invalid",
142
+ message
143
+ };
144
+ }
145
+ function unknownFlag(flag) {
146
+ if (flag[1] !== "-" && Object.hasOwn(FLAGS, `-${flag}`)) return invalid(`Unknown flag "${flag}". Use "-${flag}".`);
147
+ return invalid(`Unknown flag "${flag}". Run "mdream-crawl --help" to list the flags.`);
148
+ }
149
+ function isFlag(arg) {
150
+ return arg.startsWith("--") || Object.hasOwn(SHORT_FLAGS, arg);
151
+ }
152
+ function parseCrawlArgs(argv) {
153
+ if (argv.length === 0) return { _tag: "Interactive" };
154
+ if (argv.includes("--help") || argv.includes("-h")) return { _tag: "Help" };
155
+ if (argv.includes("--version")) return { _tag: "Version" };
156
+ const options = {};
157
+ let singlePage = false;
158
+ let positionalOnly = false;
159
+ for (let i = 0; i < argv.length; i++) {
160
+ const arg = argv[i];
161
+ if (positionalOnly || arg[0] !== "-") {
162
+ addUrl(options, arg);
163
+ continue;
164
+ }
165
+ if (arg === "--") {
166
+ positionalOnly = true;
167
+ continue;
168
+ }
169
+ const equals = arg.startsWith("--") ? arg.indexOf("=") : -1;
170
+ const written = equals === -1 ? arg : arg.slice(0, equals);
171
+ const name = Object.hasOwn(SHORT_FLAGS, written) ? SHORT_FLAGS[written] : written;
172
+ if (!Object.hasOwn(FLAGS, name)) return unknownFlag(written);
173
+ const spec = FLAGS[name];
174
+ if (spec._tag === "Switch") {
175
+ if (equals !== -1) return invalid(`"${written}" takes no value.`);
176
+ if (name === "--single-page") singlePage = true;
177
+ spec.apply(options);
178
+ continue;
179
+ }
180
+ let value;
181
+ if (equals !== -1) value = arg.slice(equals + 1);
182
+ else {
183
+ const next = argv[i + 1];
184
+ if (next === void 0 || isFlag(next)) return invalid(`"${written}" needs a value.`);
185
+ value = next;
186
+ i++;
187
+ }
188
+ if (spec._tag === "Number") {
189
+ const number = Number(value);
190
+ if (value.trim() === "" || Number.isNaN(number)) return invalid(`"${written}" needs a number. You gave ${JSON.stringify(value)}.`);
191
+ spec.apply(options, number);
192
+ } else spec.apply(options, value);
193
+ }
194
+ if (singlePage) options.depth = 0;
195
+ return {
196
+ _tag: "Crawl",
197
+ options
198
+ };
199
+ }
8
200
  async function loadMdreamConfig(cwd) {
9
201
  const { config } = await loadConfig({
10
202
  name: "mdream",
@@ -24,6 +216,53 @@ let runLogger = resolveLogger({ silent: isSilentArgv(process.argv.slice(2)) });
24
216
  const __dirname = dirname(fileURLToPath(import.meta.url));
25
217
  const packageJsonPath = join(__dirname, "..", "package.json");
26
218
  const version = JSON.parse(readFileSync(packageJsonPath, "utf-8")).version;
219
+ const HELP = `
220
+ @mdream/crawl v${version}
221
+
222
+ Crawl a website and generate llms.txt, llms-full.txt, and Markdown files.
223
+
224
+ Usage:
225
+ mdream-crawl [options] <url> Crawl a website
226
+ mdream-crawl Start interactive mode
227
+ npx @mdream/crawl [options] <url>
228
+
229
+ Options:
230
+ -u, --url <url> URL to crawl. Accepts glob patterns. Repeatable
231
+ -o, --output <dir> Output directory (default: ${CRAWL_DEFAULTS.output})
232
+ -d, --depth <number> Link depth from 0 to 10. 0 processes only the given URLs (default: ${CRAWL_DEFAULTS.depth})
233
+ --single-page Same as --depth 0
234
+ --driver <http|playwright> Crawler driver (default: ${CRAWL_DEFAULTS.driver})
235
+ --artifacts <list> Comma-separated list of ${CRAWL_ARTIFACTS.join(", ")} (default: all)
236
+ --origin <url> Origin for relative links in page Markdown (default: the page origin)
237
+ --site-name <name> Site name in llms.txt (default: the home page title)
238
+ --description <text> Site description in llms.txt (default: the home page description)
239
+ --max-pages <number> Maximum number of pages to fetch (default: no limit)
240
+ --crawl-delay <seconds> Delay between requests (default: the robots.txt Crawl-delay)
241
+ --exclude <pattern> Skip URLs that match a glob pattern. Repeatable
242
+ --skip-sitemap Skip sitemap and robots.txt discovery
243
+ --sitemap <url> Use this sitemap instead of auto-discovery. Repeatable
244
+ --allow-subdomains Crawl other subdomains of the same domain
245
+ --keep-boilerplate Keep repeated navigation and footers in page Markdown
246
+ --boilerplate-threshold <n> Fraction of pages that a block must appear in to count as boilerplate (default: ${CRAWL_DEFAULTS.boilerplateThreshold})
247
+ -v, --verbose Log details for each failed URL
248
+ -q, --quiet, --silent Drop all logs. Use it to keep stdout clean for JSON-RPC or MCP
249
+ -h, --help Show this help
250
+ --version Show the version
251
+
252
+ A flag can also take its value as --flag=value.
253
+ Flags override mdream.config.* values. Config values override the defaults.
254
+
255
+ Examples:
256
+ mdream-crawl harlanzw.com --artifacts "llms.txt,markdown"
257
+ mdream-crawl https://docs.example.com --depth 2 --artifacts llms-full.txt
258
+ mdream-crawl example.com --exclude "*/admin/*" --exclude "*/api/*"
259
+ mdream-crawl example.com --sitemap https://example.com/custom/sitemap.xml
260
+ mdream-crawl example.com/pricing --driver playwright --single-page
261
+ `;
262
+ function exitWithError(message) {
263
+ console.error(`Error: ${message}`);
264
+ process.exit(1);
265
+ }
27
266
  function checkOutputDirectoryPermissions(outputDir) {
28
267
  try {
29
268
  mkdirSync(outputDir, { recursive: true });
@@ -56,135 +295,91 @@ function checkOutputDirectoryPermissions(outputDir) {
56
295
  };
57
296
  }
58
297
  }
59
- async function interactiveCrawl() {
298
+ function splitUrls(input) {
299
+ return input.split(",").map((url) => url.trim()).filter(Boolean);
300
+ }
301
+ async function interactiveCrawl(config) {
60
302
  console.clear();
61
303
  p.intro(`☁️ @mdream/crawl v${version}`);
62
304
  const urlsInput = await p.text({
63
305
  message: "Enter starting URL for crawling (supports glob patterns):",
64
306
  placeholder: "e.g. docs.example.com, site.com/docs/**",
307
+ initialValue: config.urls?.join(", "),
65
308
  validate: (value) => {
66
- if (!value) return "Please enter at least one URL";
67
- const urls = value.split(",").map((url) => url.trim());
309
+ const urls = splitUrls(value ?? "");
310
+ if (urls.length === 0) return "Enter at least one URL.";
68
311
  for (const url of urls) {
69
- const globError = validateGlobPattern(url);
70
- if (globError) return globError;
71
- try {
72
- if (!parseUrlPattern(url).isGlob) try {
73
- new URL(withHttps(url));
74
- } catch {
75
- return `Invalid URL: ${withHttps(url)}`;
76
- }
77
- } catch (error) {
78
- return error instanceof Error ? error.message : "Invalid URL pattern";
79
- }
312
+ const error = validateCrawlUrl(url);
313
+ if (error) return error;
80
314
  }
81
315
  }
82
316
  });
83
- if (p.isCancel(urlsInput)) {
317
+ if (typeof urlsInput !== "string") {
84
318
  p.cancel("Operation cancelled.");
85
319
  return null;
86
320
  }
87
- const urls = urlsInput.split(",").map((url) => url.trim());
88
- let globPatterns;
89
- try {
90
- globPatterns = urls.map(parseUrlPattern);
91
- } catch (error) {
92
- p.cancel(error instanceof Error ? error.message : "Invalid URL pattern");
93
- return null;
94
- }
95
- const outputDir = "output";
96
- const crawlerOptions = await p.group({ driver: () => p.select({
97
- message: "Select crawler driver:",
98
- options: [{
99
- value: "http",
100
- label: "HTTP Crawler (Fast, for static content)",
101
- hint: "Recommended"
102
- }, {
103
- value: "playwright",
104
- label: "Playwright (Slower, supports JavaScript)"
105
- }],
106
- initialValue: "http"
107
- }) }, { onCancel: () => {
108
- p.cancel("Operation cancelled.");
109
- process.exit(0);
110
- } });
111
- const advancedOptions = await p.group({
112
- outputFormats: () => p.multiselect({
113
- message: "Select output formats:",
321
+ const answers = await p.group({
322
+ driver: () => p.select({
323
+ message: "Select crawler driver:",
324
+ options: [{
325
+ value: "http",
326
+ label: "HTTP Crawler (Fast, for static content)",
327
+ hint: "Recommended"
328
+ }, {
329
+ value: "playwright",
330
+ label: "Playwright (Slower, supports JavaScript)"
331
+ }],
332
+ initialValue: config.driver ?? CRAWL_DEFAULTS.driver
333
+ }),
334
+ artifacts: () => p.multiselect({
335
+ message: "Select artifacts:",
114
336
  options: [
115
337
  {
116
338
  value: "llms.txt",
117
- label: "llms.txt (basic format)",
339
+ label: "llms.txt (link index)",
118
340
  hint: "Recommended"
119
341
  },
120
342
  {
121
343
  value: "llms-full.txt",
122
- label: "llms-full.txt (extended format)"
344
+ label: "llms-full.txt (full content)"
123
345
  },
124
346
  {
125
347
  value: "markdown",
126
- label: "Individual Markdown files"
348
+ label: "Markdown file for each page"
127
349
  }
128
350
  ],
129
- initialValues: [
130
- "llms.txt",
131
- "llms-full.txt",
132
- "markdown"
133
- ]
351
+ initialValues: config.artifacts ?? [...CRAWL_DEFAULTS.artifacts]
134
352
  }),
135
353
  skipSitemap: () => p.confirm({
136
354
  message: "Skip sitemap.xml and robots.txt discovery?",
137
- initialValue: false
355
+ initialValue: config.skipSitemap ?? false
138
356
  })
139
357
  }, { onCancel: () => {
140
358
  p.cancel("Operation cancelled.");
141
359
  process.exit(0);
142
360
  } });
143
- const firstUrl = urls[0];
144
- const inferredOrigin = (() => {
145
- try {
146
- const url = new URL(withHttps(firstUrl));
147
- return `${url.protocol}//${url.host}`;
148
- } catch {
149
- return;
150
- }
151
- })();
152
- const outputFormats = advancedOptions.outputFormats.map((f) => {
153
- switch (f) {
154
- case "llms.txt": return "llms.txt";
155
- case "llms-full.txt": return "llms-full.txt";
156
- case "markdown": return "Individual MD files";
157
- default: return f;
158
- }
159
- });
160
- const summary = [
161
- `URLs: ${urls.join(", ")}`,
162
- `Output: ${relative(process.cwd(), outputDir) || outputDir}`,
163
- `Driver: ${crawlerOptions.driver}`,
164
- `Max pages: Unlimited`,
165
- `Follow links: Yes (depth 3)`,
166
- `Output formats: ${outputFormats.join(", ")}`,
167
- `Sitemap discovery: ${advancedOptions.skipSitemap ? "Skipped" : "Automatic"}`,
168
- inferredOrigin && `Origin: ${inferredOrigin}`
169
- ].filter(Boolean);
170
- p.note(summary.join("\n"), "Crawl Configuration");
171
- if (advancedOptions.skipSitemap && globPatterns.some((p) => p.isGlob)) p.log.warn("Warning: Using --skip-sitemap with glob URLs may not discover all matching pages.");
172
361
  return {
173
- urls,
174
- outputDir: resolve(outputDir),
175
- driver: crawlerOptions.driver,
176
- maxRequestsPerCrawl: Number.MAX_SAFE_INTEGER,
177
- followLinks: true,
178
- generateLlmsTxt: advancedOptions.outputFormats.includes("llms.txt"),
179
- generateLlmsFullTxt: advancedOptions.outputFormats.includes("llms-full.txt"),
180
- generateIndividualMd: advancedOptions.outputFormats.includes("markdown"),
181
- origin: inferredOrigin,
182
- globPatterns,
183
- verbose: false,
184
- maxDepth: 3,
185
- skipSitemap: advancedOptions.skipSitemap
362
+ urls: splitUrls(urlsInput),
363
+ driver: answers.driver,
364
+ artifacts: answers.artifacts,
365
+ skipSitemap: answers.skipSitemap
186
366
  };
187
367
  }
368
+ function summarize(options) {
369
+ const maxPages = Number.isFinite(options.maxPages) ? String(options.maxPages) : "no limit";
370
+ return [
371
+ `URL: ${options.urls.join(", ")}`,
372
+ `Output: ${relative(process.cwd(), options.output) || "."}`,
373
+ `Driver: ${options.driver} · Depth: ${options.depth} · Max pages: ${maxPages}`,
374
+ `Artifacts: ${options.artifacts.length > 0 ? options.artifacts.join(", ") : "none"}`,
375
+ options.origin && `Origin: ${options.origin}`,
376
+ options.exclude.length > 0 && `Exclude: ${options.exclude.join(", ")}`,
377
+ options.skipSitemap && `Skip sitemap: yes`,
378
+ options.sitemap.length > 0 && `Sitemap: ${options.sitemap.join(", ")}`,
379
+ options.allowSubdomains && `Allow subdomains: yes`,
380
+ options.verbose && `Verbose: on`
381
+ ].filter(Boolean).join("\n");
382
+ }
188
383
  async function showCrawlResults(logger, successful, failed, outputDir, generatedFiles, durationSeconds, latency) {
189
384
  let line = `${successful} pages in ${`${durationSeconds.toFixed(1)}s`}`;
190
385
  if (failed > 0) line += ` (${failed} failed)`;
@@ -192,261 +387,45 @@ async function showCrawlResults(logger, successful, failed, outputDir, generated
192
387
  const avg = Math.round(latency.total / latency.count);
193
388
  const min = latency.min === Infinity ? 0 : Math.round(latency.min);
194
389
  const max = Math.round(latency.max);
195
- line += ` \u00B7 HTTP Latency: avg ${avg}ms, min ${min}ms, max ${max}ms`;
390
+ line += ` · HTTP Latency: avg ${avg}ms, min ${min}ms, max ${max}ms`;
196
391
  }
197
392
  logger.success(line);
198
- logger.info(`${generatedFiles.join(", ")} \u2192 ${relative(process.cwd(), outputDir) || "."}`);
393
+ if (generatedFiles.length > 0) logger.info(`${generatedFiles.join(", ")} → ${relative(process.cwd(), outputDir) || "."}`);
199
394
  }
200
- function parseCliArgs() {
201
- const args = process.argv.slice(2);
202
- const silent = isSilentArgv(args);
203
- const logger = resolveLogger({ silent });
204
- if (args.includes("--help") || args.includes("-h")) {
205
- console.log(`
206
- @mdream/crawl v${version}
207
-
208
- Multi-page website crawler that generates comprehensive llms.txt files
209
-
210
- Usage:
211
- @mdream/crawl [options] <url> Crawl a website with CLI flags
212
- @mdream/crawl Start interactive mode
213
-
214
- Options:
215
- -u, --url <url> Website URL to crawl
216
- -o, --output <dir> Output directory (default: output)
217
- -d, --depth <number> Crawl depth, 0 for single page (default: 3)
218
- --single-page Only process the given URL(s), no crawling (alias for --depth 0)
219
- --driver <http|playwright> Crawler driver (default: http)
220
- --artifacts <list> Comma-separated list of artifacts: llms.txt,llms-full.txt,markdown (default: all)
221
- --origin <url> Origin URL for resolving relative paths (overrides auto-detection)
222
- --site-name <name> Override site name (overrides auto-extracted title)
223
- --description <desc> Override site description (overrides auto-extracted description)
224
- --max-pages <number> Maximum pages to crawl (default: unlimited)
225
- --crawl-delay <seconds> Crawl delay in seconds
226
- --exclude <pattern> Exclude URLs matching glob patterns (can be used multiple times)
227
- --skip-sitemap Skip sitemap.xml and robots.txt discovery
228
- --sitemap <url> Use an explicit sitemap URL instead of auto-discovery (repeatable for multi-part sitemaps)
229
- --allow-subdomains Crawl across subdomains of the same root domain
230
- --keep-boilerplate Keep repeated site chrome (nav/footer) in per-page output (default: stripped)
231
- --boilerplate-threshold <n> Fraction of pages a block must repeat in to count as chrome (0-1, default: ${DEFAULT_BOILERPLATE_THRESHOLD})
232
- -v, --verbose Enable verbose logging
233
- -q, --quiet, --silent Suppress all logs (clean stdout for JSON-RPC/MCP)
234
- -h, --help Show this help message
235
- --version Show version number
236
-
237
- Note: Sitemap discovery and robots.txt checking are automatic unless --skip-sitemap is used.
238
-
239
- Examples:
240
- @mdream/crawl -u harlanzw.com --artifacts "llms.txt,markdown"
241
- @mdream/crawl --url https://docs.example.com --depth 2 --artifacts "llms-full.txt"
242
- @mdream/crawl -u example.com --exclude "*/admin/*" --exclude "*/api/*"
243
- @mdream/crawl -u example.com --verbose
244
- @mdream/crawl -u example.com --skip-sitemap
245
- @mdream/crawl -u example.com --sitemap https://example.com/custom/sitemap.xml
246
- @mdream/crawl -u example.com --driver playwright --single-page
247
- `);
395
+ async function main() {
396
+ const command = parseCrawlArgs(process.argv.slice(2));
397
+ if (command._tag === "Help") {
398
+ console.log(HELP);
248
399
  process.exit(0);
249
400
  }
250
- if (args.includes("--version")) {
401
+ if (command._tag === "Version") {
251
402
  console.log(`@mdream/crawl v${version}`);
252
403
  process.exit(0);
253
404
  }
254
- if (args.length === 0) return null;
255
- const getArgValue = (flag) => {
256
- const index = args.findIndex((arg) => arg === flag || arg === flag.replace("--", "-"));
257
- return index >= 0 && index + 1 < args.length ? args[index + 1] : void 0;
258
- };
259
- const getArgValues = (flag) => {
260
- const values = [];
261
- for (let i = 0; i < args.length; i++) if (args[i] === flag || args[i] === flag.replace("--", "-")) {
262
- if (i + 1 < args.length && !args[i + 1].startsWith("-")) values.push(args[i + 1]);
263
- }
264
- return values;
265
- };
266
- const urlFromFlag = getArgValue("--url") || getArgValue("-u");
267
- const urlFromArgs = args.find((arg) => !arg.startsWith("-") && !args[args.indexOf(arg) - 1]?.startsWith("-"));
268
- const url = urlFromFlag || urlFromArgs;
269
- if (!url) {
270
- logger.error("Error: URL is required when using CLI arguments");
271
- logger.info("Use --help for usage information or run without arguments for interactive mode");
272
- process.exit(1);
273
- }
274
- const globError = validateGlobPattern(url);
275
- if (globError) {
276
- logger.error(`Error: ${globError}`);
277
- process.exit(1);
278
- }
279
- let parsed;
405
+ if (command._tag === "Invalid") exitWithError(command.message);
406
+ const fileConfig = await loadMdreamConfig();
280
407
  try {
281
- parsed = parseUrlPattern(url);
408
+ assertCrawlOptionKeys(fileConfig);
282
409
  } catch (error) {
283
- logger.error(`Error: ${error instanceof Error ? error.message : "Invalid URL pattern"}`);
284
- process.exit(1);
285
- }
286
- if (!parsed.isGlob) try {
287
- new URL(withHttps(url));
288
- } catch {
289
- logger.error(`Error: Invalid URL: ${withHttps(url)}`);
290
- process.exit(1);
291
- }
292
- const excludePatterns = getArgValues("--exclude");
293
- for (const pattern of excludePatterns) {
294
- const excludeError = validateGlobPattern(pattern);
295
- if (excludeError) {
296
- logger.error(`Error in exclude pattern: ${excludeError}`);
297
- process.exit(1);
298
- }
299
- }
300
- const depthStr = args.includes("--single-page") ? "0" : getArgValue("--depth") || getArgValue("-d") || "3";
301
- const depth = Number(depthStr);
302
- if (!Number.isInteger(depth) || depth < 0 || depth > 10) {
303
- logger.error("Error: Depth must be an integer between 0 and 10");
304
- process.exit(1);
305
- }
306
- const driver = getArgValue("--driver");
307
- if (driver && driver !== "http" && driver !== "playwright") {
308
- logger.error("Error: Driver must be either \"http\" or \"playwright\"");
309
- process.exit(1);
410
+ exitWithError(`mdream.config: ${error instanceof Error ? error.message : String(error)}`);
310
411
  }
311
- const maxPagesStr = getArgValue("--max-pages");
312
- if (maxPagesStr) {
313
- const maxPages = Number.parseInt(maxPagesStr);
314
- if (Number.isNaN(maxPages) || maxPages < 1) {
315
- logger.error("Error: Max pages must be a positive number");
316
- process.exit(1);
317
- }
318
- }
319
- const crawlDelayStr = getArgValue("--crawl-delay");
320
- if (crawlDelayStr) {
321
- const crawlDelay = Number.parseInt(crawlDelayStr);
322
- if (Number.isNaN(crawlDelay) || crawlDelay < 0) {
323
- logger.error("Error: Crawl delay must be a non-negative number");
324
- process.exit(1);
325
- }
326
- }
327
- const artifactsStr = getArgValue("--artifacts");
328
- const artifacts = artifactsStr ? artifactsStr.split(",").map((a) => a.trim()) : [
329
- "llms.txt",
330
- "llms-full.txt",
331
- "markdown"
332
- ];
333
- const validArtifacts = [
334
- "llms.txt",
335
- "llms-full.txt",
336
- "markdown"
337
- ];
338
- for (const artifact of artifacts) if (!validArtifacts.includes(artifact)) {
339
- logger.error(`Error: Invalid artifact '${artifact}'. Valid options: ${validArtifacts.join(", ")}`);
340
- process.exit(1);
341
- }
342
- const originOverride = getArgValue("--origin");
343
- const inferredOrigin = (() => {
344
- if (originOverride) return originOverride;
345
- try {
346
- const urlObj = new URL(withHttps(url));
347
- return `${urlObj.protocol}//${urlObj.host}`;
348
- } catch {
349
- return;
350
- }
351
- })();
352
- const siteNameOverride = getArgValue("--site-name");
353
- const descriptionOverride = getArgValue("--description");
354
- const patterns = [parsed];
355
- const verbose = args.includes("--verbose") || args.includes("-v");
356
- const skipSitemap = args.includes("--skip-sitemap");
357
- const sitemapUrls = getArgValues("--sitemap").map((u) => withHttps(u));
358
- if (args.filter((a) => a === "--sitemap" || a === "-sitemap").length > sitemapUrls.length) {
359
- logger.error("Error: --sitemap requires a URL value");
360
- process.exit(1);
361
- }
362
- for (const sitemapUrl of sitemapUrls) try {
363
- new URL(sitemapUrl);
364
- } catch {
365
- logger.error(`Error: Invalid sitemap URL: ${sitemapUrl}`);
366
- process.exit(1);
367
- }
368
- const allowSubdomains = args.includes("--allow-subdomains");
369
- const stripBoilerplate = args.includes("--keep-boilerplate") ? false : void 0;
370
- const boilerplateThresholdStr = getArgValue("--boilerplate-threshold");
371
- let boilerplateThreshold;
372
- if (boilerplateThresholdStr !== void 0) {
373
- boilerplateThreshold = Number(boilerplateThresholdStr);
374
- if (Number.isNaN(boilerplateThreshold) || boilerplateThreshold <= 0 || boilerplateThreshold > 1) {
375
- logger.error("Error: Boilerplate threshold must be a number between 0 (exclusive) and 1");
376
- process.exit(1);
377
- }
412
+ const interactive = command._tag === "Interactive";
413
+ const layer = interactive ? await interactiveCrawl(fileConfig) : command.options;
414
+ if (!layer) process.exit(0);
415
+ const input = mergeCrawlOptions(fileConfig, layer);
416
+ if (!input.urls || input.urls.length === 0) exitWithError("A URL is required. Pass it as an argument, with --url, or as \"urls\" in mdream.config. Run without arguments for interactive mode.");
417
+ let options;
418
+ try {
419
+ options = resolveCrawlOptions(input);
420
+ } catch (error) {
421
+ exitWithError(error instanceof Error ? error.message : String(error));
378
422
  }
379
- if (skipSitemap && parsed.isGlob) logger.warn("Warning: Using --skip-sitemap with glob URLs may not discover all matching pages.");
380
- return {
381
- urls: [url],
382
- outputDir: resolve(getArgValue("--output") || getArgValue("-o") || "output"),
383
- driver: driver || "http",
384
- maxRequestsPerCrawl: Number.parseInt(maxPagesStr || String(Number.MAX_SAFE_INTEGER)),
385
- followLinks: depth > 0,
386
- maxDepth: depth,
387
- generateLlmsTxt: artifacts.includes("llms.txt"),
388
- generateLlmsFullTxt: artifacts.includes("llms-full.txt"),
389
- generateIndividualMd: artifacts.includes("markdown"),
390
- siteNameOverride,
391
- descriptionOverride,
392
- origin: inferredOrigin,
393
- globPatterns: patterns,
394
- crawlDelay: crawlDelayStr ? Number.parseInt(crawlDelayStr) : void 0,
395
- exclude: excludePatterns.length > 0 ? excludePatterns : void 0,
396
- verbose,
397
- skipSitemap,
398
- sitemapUrls: sitemapUrls.length > 0 ? sitemapUrls : void 0,
399
- allowSubdomains,
400
- stripBoilerplate,
401
- boilerplateThreshold,
402
- silent
403
- };
404
- }
405
- async function main() {
406
- const cliOptions = parseCliArgs();
407
- const fileConfig = await loadMdreamConfig();
408
- const silent = (cliOptions?.silent ?? false) || (fileConfig.silent ?? false);
409
- const logger = resolveLogger({ silent });
423
+ const logger = options.logger;
410
424
  runLogger = logger;
411
- let options;
412
- if (cliOptions) {
413
- const configExclude = fileConfig.exclude || [];
414
- const cliExclude = cliOptions.exclude || [];
415
- options = {
416
- ...cliOptions,
417
- driver: cliOptions.driver || fileConfig.driver || "http",
418
- maxDepth: cliOptions.maxDepth ?? fileConfig.maxDepth,
419
- crawlDelay: cliOptions.crawlDelay ?? fileConfig.crawlDelay,
420
- skipSitemap: cliOptions.skipSitemap || fileConfig.skipSitemap || false,
421
- sitemapUrls: cliOptions.sitemapUrls ?? (fileConfig.sitemap ? Array.isArray(fileConfig.sitemap) ? fileConfig.sitemap : [fileConfig.sitemap] : void 0),
422
- allowSubdomains: cliOptions.allowSubdomains || fileConfig.allowSubdomains || false,
423
- stripBoilerplate: cliOptions.stripBoilerplate ?? fileConfig.stripBoilerplate ?? true,
424
- boilerplateThreshold: cliOptions.boilerplateThreshold ?? fileConfig.boilerplateThreshold,
425
- verbose: cliOptions.verbose || fileConfig.verbose || false,
426
- silent,
427
- exclude: configExclude.length > 0 || cliExclude.length > 0 ? [...configExclude, ...cliExclude] : void 0,
428
- hooks: fileConfig.hooks
429
- };
430
- logger.intro(`☁️ mdream v${version}`);
431
- const formats = [];
432
- if (options.generateLlmsTxt) formats.push("llms.txt");
433
- if (options.generateLlmsFullTxt) formats.push("llms-full.txt");
434
- if (options.generateIndividualMd) formats.push("Individual MD files");
435
- const summary = [
436
- `URL: ${options.urls.join(", ")}`,
437
- `Output: ${relative(process.cwd(), options.outputDir) || "."}`,
438
- `Driver: ${options.driver} \u00B7 Depth: ${options.maxDepth}`,
439
- `Formats: ${formats.join(", ")}`,
440
- options.exclude && options.exclude.length > 0 && `Exclude: ${options.exclude.join(", ")}`,
441
- options.skipSitemap && `Skip sitemap: Yes`,
442
- options.sitemapUrls && options.sitemapUrls.length > 0 && `Sitemap: ${options.sitemapUrls.join(", ")}`,
443
- options.allowSubdomains && `Allow subdomains: Yes`,
444
- options.verbose && `Verbose: Enabled`
445
- ].filter(Boolean);
446
- logger.note(summary.join("\n"), "Configuration");
447
- } else options = await interactiveCrawl();
448
- if (!options) process.exit(0);
449
- const permCheck = checkOutputDirectoryPermissions(options.outputDir);
425
+ if (!interactive) logger.intro(`☁️ mdream v${version}`);
426
+ logger.note(summarize(options), "Configuration");
427
+ if (options.skipSitemap && options.patterns.some((pattern) => pattern.isGlob)) logger.warn("Sitemap discovery is off. A glob URL can miss pages that no link reaches.");
428
+ const permCheck = checkOutputDirectoryPermissions(options.output);
450
429
  if (!permCheck.success) {
451
430
  logger.error(permCheck.error);
452
431
  if (permCheck.error?.includes("Permission denied")) logger.info("Tip: Try running with elevated privileges (e.g., sudo) or change the output directory permissions.");
@@ -460,8 +439,11 @@ async function main() {
460
439
  process.exit(1);
461
440
  }
462
441
  const { ensurePlaywrightInstalled, isUseChromeSupported } = await import("./_chunks/playwright-utils.mjs");
463
- if (await isUseChromeSupported()) {
464
- options.useChrome = true;
442
+ if (input.useChrome !== false && await isUseChromeSupported()) {
443
+ options = {
444
+ ...options,
445
+ useChrome: true
446
+ };
465
447
  logger.info("System Chrome detected and enabled.");
466
448
  } else {
467
449
  if (!await ensurePlaywrightInstalled(logger)) {
@@ -476,7 +458,7 @@ async function main() {
476
458
  const startTime = Date.now();
477
459
  let crawlStartTime = 0;
478
460
  let lastProgress;
479
- const results = await crawlAndGenerate(options, (progress) => {
461
+ const results = await runCrawl(options, (progress) => {
480
462
  lastProgress = progress;
481
463
  if (progress.sitemap.status === "discovering") s.message("Discovering sitemaps");
482
464
  else if (progress.sitemap.status === "processing") s.message(`Processing sitemap... Found ${progress.sitemap.found} URLs`);
@@ -488,34 +470,22 @@ async function main() {
488
470
  const elapsed = (Date.now() - crawlStartTime) / 1e3;
489
471
  const rate = elapsed > .1 ? Math.round(processed / elapsed) : 0;
490
472
  let msg = processed > total ? `Crawling ${processed} pages` : `Crawling ${processed}/${total}`;
491
- if (rate > 0) msg += ` \u00B7 ${rate}/s`;
492
- if (failed > 0) msg += ` \u00B7 ${failed} failed`;
473
+ if (rate > 0) msg += ` · ${rate}/s`;
474
+ if (failed > 0) msg += ` · ${failed} failed`;
493
475
  s.message(msg);
494
476
  } else if (progress.generation.status === "generating") s.message(progress.generation.current || "Generating files");
495
477
  });
496
478
  s.stop();
497
479
  const durationSeconds = (Date.now() - startTime) / 1e3;
498
480
  const successful = results.filter((r) => r.success).length;
499
- const failed = results.filter((r) => !r.success).length;
500
481
  const failedResults = results.filter((r) => !r.success);
501
- if (failed > 0 && cliOptions) {
482
+ if (failedResults.length > 0) {
502
483
  logger.error("Failed URLs:");
503
- failedResults.forEach((result) => {
504
- logger.error(` ${result.url}: ${result.error || "Unknown error"}`);
505
- });
506
- } else if (failed > 0) {
507
- console.log("\nFailed URLs:");
508
- failedResults.forEach((result) => {
509
- console.log(` - ${result.url}: ${result.error || "Unknown error"}`);
510
- });
484
+ for (const result of failedResults) logger.error(` ${result.url}: ${result.error || "Unknown error"}`);
511
485
  }
512
486
  const generatedFiles = [];
513
- if (successful > 0) {
514
- if (options.generateLlmsTxt) generatedFiles.push("llms.txt");
515
- if (options.generateLlmsFullTxt) generatedFiles.push("llms-full.txt");
516
- if (options.generateIndividualMd) generatedFiles.push(`${successful} MD files`);
517
- }
518
- await showCrawlResults(logger, successful, failed, options.outputDir, generatedFiles, durationSeconds, lastProgress?.crawling.latency);
487
+ if (successful > 0) for (const artifact of options.artifacts) generatedFiles.push(artifact === "markdown" ? `${successful} MD files` : artifact);
488
+ await showCrawlResults(logger, successful, failedResults.length, options.output, generatedFiles, durationSeconds, lastProgress?.crawling.latency);
519
489
  process.exit(0);
520
490
  }
521
491
  main().catch((error) => {