@mdream/crawl 1.4.1 → 1.5.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -49,6 +49,7 @@ npx @mdream/crawl -u https://docs.example.com
49
49
  | `--crawl-delay <seconds>` | | Delay between requests in seconds | from `robots.txt` or none |
50
50
  | `--exclude <pattern>` | | Exclude URLs matching glob patterns (repeatable) | none |
51
51
  | `--skip-sitemap` | | Skip `sitemap.xml` and `robots.txt` discovery | `false` |
52
+ | `--sitemap <url>` | | Use an explicit sitemap URL instead of auto-discovery (repeatable for multi-part sitemaps) | auto-discovered |
52
53
  | `--allow-subdomains` | | Crawl across subdomains of the same root domain | `false` |
53
54
  | `--verbose` | `-v` | Enable verbose logging | `false` |
54
55
  | `--help` | `-h` | Show help message | |
@@ -131,6 +132,7 @@ const results = await crawlAndGenerate({
131
132
  | `exclude` | `string[]` | `[]` | Glob patterns for URLs to exclude |
132
133
  | `crawlDelay` | `number` | from `robots.txt` | Delay between requests in seconds |
133
134
  | `skipSitemap` | `boolean` | `false` | Skip `sitemap.xml` and `robots.txt` discovery |
135
+ | `sitemapUrls` | `string[]` | auto-discovered | Explicit sitemap URL(s) to use instead of auto-discovery. Supports non-standard locations and multiple parts (all loaded and merged). Ignored when `skipSitemap` is set |
134
136
  | `allowSubdomains` | `boolean` | `false` | Crawl across subdomains of the same root domain (e.g. `docs.example.com` + `blog.example.com`). Output files are namespaced by hostname to avoid collisions |
135
137
  | `useChrome` | `boolean` | `false` | Use system Chrome instead of Playwright's bundled browser (Playwright driver only) |
136
138
  | `chunkSize` | `number` | | Chunk size passed to mdream for markdown conversion |
@@ -294,6 +296,7 @@ CLI arguments override config file values. Array options like `exclude` are conc
294
296
  | `maxPages` | `number` | Maximum pages to crawl |
295
297
  | `crawlDelay` | `number` | Delay between requests (seconds) |
296
298
  | `skipSitemap` | `boolean` | Skip sitemap discovery |
299
+ | `sitemap` | `string \| string[]` | Explicit sitemap URL(s) to use instead of auto-discovery |
297
300
  | `allowSubdomains` | `boolean` | Crawl across subdomains |
298
301
  | `verbose` | `boolean` | Enable verbose logging |
299
302
  | `artifacts` | `string[]` | Output formats: `llms.txt`, `llms-full.txt`, `markdown` |
@@ -437,16 +440,47 @@ Waits for `networkidle` before extracting content. Automatically detects and use
437
440
  By default, the crawler performs sitemap discovery before crawling:
438
441
 
439
442
  1. Fetches `robots.txt` to find `Sitemap:` directives and `Crawl-delay` values
440
- 2. Loads sitemaps referenced in `robots.txt`
443
+ 2. Loads sitemaps referenced in `robots.txt` (authoritative, tried first)
441
444
  3. Falls back to `/sitemap.xml`
442
445
  4. Tries common alternatives: `/sitemap_index.xml`, `/sitemaps.xml`, `/sitemap-index.xml`
443
- 5. Supports sitemap index files (recursively loads child sitemaps)
446
+ 5. Supports sitemap index files (recursively loads child sitemaps, with cycle protection)
444
447
  6. Filters discovered URLs against glob patterns and exclusion rules
445
448
 
449
+ The first candidate that yields matching URLs wins, so a `robots.txt` sitemap is never overwritten by a well-known fallback. `<loc>` values are CDATA-unwrapped and XML-entity-decoded (`&amp;` → `&`).
450
+
446
451
  The home page is always included for metadata extraction (site name, description).
447
452
 
448
453
  Disable with `--skip-sitemap` or `skipSitemap: true`.
449
454
 
455
+ ### Overriding the sitemap location
456
+
457
+ When a site keeps its sitemap at a non-standard URL, or splits it across several links, pin the location(s) explicitly instead of relying on discovery:
458
+
459
+ ```bash
460
+ # Single non-standard location
461
+ npx @mdream/crawl -u example.com --sitemap https://example.com/custom/sitemap.xml
462
+
463
+ # Multiple parts (repeatable), all loaded and merged
464
+ npx @mdream/crawl -u example.com \
465
+ --sitemap https://example.com/sitemap-posts.xml \
466
+ --sitemap https://example.com/sitemap-pages.xml
467
+ ```
468
+
469
+ Or in `mdream.config.ts`:
470
+
471
+ ```ts
472
+ import { defineConfig } from '@mdream/crawl'
473
+
474
+ export default defineConfig({
475
+ sitemap: [
476
+ 'https://example.com/sitemap-posts.xml',
477
+ 'https://example.com/sitemap-pages.xml',
478
+ ],
479
+ })
480
+ ```
481
+
482
+ An explicit sitemap replaces auto-discovery entirely (`robots.txt` is still read for `Crawl-delay`).
483
+
450
484
  ## Output Formats
451
485
 
452
486
  ### Individual Markdown Files
@@ -9,6 +9,113 @@ import { withHttps } from "ufo";
9
9
  import picomatch from "picomatch";
10
10
  import { getDomain } from "tldts";
11
11
  import * as p from "@clack/prompts";
12
+ const DEFAULT_BOILERPLATE_THRESHOLD = .5;
13
+ const TOKEN_RE = /\S+/g;
14
+ function hashToken(s, start, end) {
15
+ let h = 2166136261;
16
+ for (let i = start; i < end; i++) {
17
+ h ^= s.charCodeAt(i);
18
+ h = Math.imul(h, 16777619) >>> 0;
19
+ }
20
+ return h >>> 0;
21
+ }
22
+ function tokenize(md, k) {
23
+ const starts = [];
24
+ const ends = [];
25
+ const tokenHashes = [];
26
+ TOKEN_RE.lastIndex = 0;
27
+ let m;
28
+ while ((m = TOKEN_RE.exec(md)) !== null) {
29
+ const start = m.index;
30
+ const end = TOKEN_RE.lastIndex;
31
+ starts.push(start);
32
+ ends.push(end);
33
+ tokenHashes.push(hashToken(md, start, end));
34
+ }
35
+ const n = tokenHashes.length;
36
+ const shingleHashes = [];
37
+ if (n === 0) return {
38
+ starts,
39
+ ends,
40
+ tokenHashes,
41
+ shingleHashes
42
+ };
43
+ if (n < k) shingleHashes.push(mixHashes(tokenHashes, 0, n));
44
+ else for (let i = 0; i + k <= n; i++) shingleHashes.push(mixHashes(tokenHashes, i, i + k));
45
+ return {
46
+ starts,
47
+ ends,
48
+ tokenHashes,
49
+ shingleHashes
50
+ };
51
+ }
52
+ function mixHashes(tokenHashes, from, to) {
53
+ let h = 2166136261;
54
+ for (let i = from; i < to; i++) {
55
+ h ^= tokenHashes[i];
56
+ h = Math.imul(h, 16777619) >>> 0;
57
+ }
58
+ return h >>> 0;
59
+ }
60
+ function detectChromeShingles(docs, threshold) {
61
+ const counts = /* @__PURE__ */ new Map();
62
+ for (let d = 0; d < docs.length; d++) {
63
+ const shingles = docs[d].shingleHashes;
64
+ const seen = /* @__PURE__ */ new Set();
65
+ for (let i = 0; i < shingles.length; i++) {
66
+ const h = shingles[i];
67
+ if (seen.has(h)) continue;
68
+ seen.add(h);
69
+ counts.set(h, (counts.get(h) ?? 0) + 1);
70
+ }
71
+ }
72
+ const minCount = Math.max(2, Math.ceil(threshold * docs.length));
73
+ const chrome = /* @__PURE__ */ new Set();
74
+ for (const [h, c] of counts) if (c >= minCount) chrome.add(h);
75
+ return chrome;
76
+ }
77
+ function stripDoc(md, doc, chrome, k, minRun) {
78
+ const n = doc.tokenHashes.length;
79
+ if (n === 0) return md;
80
+ const covered = new Uint8Array(n);
81
+ const shingles = doc.shingleHashes;
82
+ if (n < k) {
83
+ if (shingles.length === 1 && chrome.has(shingles[0])) covered.fill(1);
84
+ } else for (let i = 0; i < shingles.length; i++) if (chrome.has(shingles[i])) for (let j = i; j < i + k; j++) covered[j] = 1;
85
+ const keptRanges = [];
86
+ let runStart = 0;
87
+ let i = 0;
88
+ while (i < n) {
89
+ if (!covered[i]) {
90
+ i++;
91
+ continue;
92
+ }
93
+ let j = i;
94
+ while (j < n && covered[j]) j++;
95
+ if (j - i >= minRun) {
96
+ if (i > runStart) keptRanges.push([runStart, i]);
97
+ runStart = j;
98
+ }
99
+ i = j;
100
+ }
101
+ if (runStart < n) keptRanges.push([runStart, n]);
102
+ if (keptRanges.length === 1 && keptRanges[0][0] === 0 && keptRanges[0][1] === n) return md;
103
+ if (keptRanges.length === 0) return md;
104
+ const pieces = [];
105
+ for (const [s, e] of keptRanges) pieces.push(md.slice(doc.starts[s], doc.ends[e - 1]).trim());
106
+ return pieces.filter(Boolean).join("\n\n");
107
+ }
108
+ function stripBoilerplateFromCorpus(contents, options = {}) {
109
+ const threshold = options.threshold ?? .5;
110
+ const minDocs = options.minDocs ?? 3;
111
+ const k = Math.max(2, options.shingleSize ?? 6);
112
+ const minRun = Math.max(k, options.minRun ?? 12);
113
+ if (contents.length < minDocs) return contents.slice();
114
+ const docs = contents.map((md) => tokenize(md, k));
115
+ const chrome = detectChromeShingles(docs, threshold);
116
+ if (chrome.size === 0) return contents.slice();
117
+ return contents.map((md, idx) => stripDoc(md, docs[idx], chrome, k, minRun));
118
+ }
12
119
  function stripGlobTail(s) {
13
120
  const idx = s.indexOf("*");
14
121
  return idx === -1 ? s : s.slice(0, idx);
@@ -134,7 +241,7 @@ const ROBOTS_CRAWL_DELAY_RE = /Crawl-delay:\s*(\d+(?:\.\d+)?)/i;
134
241
  const URL_TRAILING_SLASH_RE = /\/$/;
135
242
  const URL_PATH_UNSAFE_CHARS_RE = /[^\w\-]/g;
136
243
  const FRONTMATTER_BLOCK_RE = /^---[^\n]*\n[\s\S]*?\n---[^\n]*\n?/;
137
- const TRACKING_PARAMS = new Set([
244
+ const TRACKING_PARAMS = /* @__PURE__ */ new Set([
138
245
  "utm_source",
139
246
  "utm_medium",
140
247
  "utm_campaign",
@@ -191,6 +298,23 @@ function extractCdataUrl(url) {
191
298
  if (url.startsWith("<![CDATA[") && url.endsWith("]]>")) return url.slice(9, -3);
192
299
  return url;
193
300
  }
301
+ const XML_ENTITIES = {
302
+ "&lt;": "<",
303
+ "&gt;": ">",
304
+ "&quot;": "\"",
305
+ "&apos;": "'",
306
+ "&amp;": "&"
307
+ };
308
+ const XML_ENTITY_RE = /&(?:lt|gt|quot|apos|amp);/g;
309
+ function decodeXmlEntities(value) {
310
+ if (!value.includes("&")) return value;
311
+ return value.replace(XML_ENTITY_RE, (m) => XML_ENTITIES[m]);
312
+ }
313
+ function cleanLoc(raw) {
314
+ const value = raw.trim();
315
+ if (value.startsWith("<![CDATA[") && value.endsWith("]]>")) return extractCdataUrl(value);
316
+ return decodeXmlEntities(value);
317
+ }
194
318
  function normalizeUrl(resolved) {
195
319
  resolved.hash = "";
196
320
  const params = resolved.searchParams;
@@ -198,7 +322,9 @@ function normalizeUrl(resolved) {
198
322
  params.sort();
199
323
  return resolved.href.replace(URL_TRAILING_SLASH_RE, "") || resolved.href;
200
324
  }
201
- async function loadSitemap(sitemapUrl) {
325
+ async function loadSitemap(sitemapUrl, visited = /* @__PURE__ */ new Set()) {
326
+ if (visited.has(sitemapUrl)) return [];
327
+ visited.add(sitemapUrl);
202
328
  const xmlContent = await ofetch(sitemapUrl, {
203
329
  headers: FETCH_HEADERS,
204
330
  timeout: 1e4,
@@ -213,9 +339,9 @@ async function loadSitemap(sitemapUrl) {
213
339
  while (true) {
214
340
  match = SITEMAP_INDEX_LOC_RE.exec(xmlContent);
215
341
  if (match === null) break;
216
- childSitemaps.push(extractCdataUrl(match[1]));
342
+ childSitemaps.push(cleanLoc(match[1]));
217
343
  }
218
- const childResults = await Promise.allSettled(childSitemaps.map((url) => loadSitemap(url)));
344
+ const childResults = await Promise.allSettled(childSitemaps.map((url) => loadSitemap(url, visited)));
219
345
  const allUrls = [];
220
346
  for (const result of childResults) if (result.status === "fulfilled") allUrls.push(...result.value);
221
347
  return allUrls;
@@ -226,7 +352,7 @@ async function loadSitemap(sitemapUrl) {
226
352
  while (true) {
227
353
  match = SITEMAP_URL_LOC_RE.exec(xmlContent);
228
354
  if (match === null) break;
229
- urls.push(extractCdataUrl(match[1]));
355
+ urls.push(cleanLoc(match[1]));
230
356
  }
231
357
  return urls;
232
358
  }
@@ -291,7 +417,7 @@ async function runConcurrent(items, concurrency, fn) {
291
417
  await Promise.all(workers);
292
418
  }
293
419
  async function crawlAndGenerate(options, onProgress) {
294
- const { urls, outputDir: rawOutputDir, maxRequestsPerCrawl = Number.MAX_SAFE_INTEGER, generateLlmsTxt = true, generateLlmsFullTxt = false, generateIndividualMd = true, origin, driver = "http", useChrome, followLinks = false, maxDepth = 1, globPatterns = [], crawlDelay: userCrawlDelay, exclude = [], siteNameOverride, descriptionOverride, verbose = false, skipSitemap = false, allowSubdomains = false, hooks: hooksConfig, onPage } = options;
420
+ const { urls, outputDir: rawOutputDir, maxRequestsPerCrawl = Number.MAX_SAFE_INTEGER, generateLlmsTxt = true, generateLlmsFullTxt = false, generateIndividualMd = true, origin, driver = "http", useChrome, followLinks = false, maxDepth = 1, globPatterns = [], crawlDelay: userCrawlDelay, exclude = [], siteNameOverride, descriptionOverride, verbose = false, skipSitemap = false, sitemapUrls: explicitSitemapUrls, allowSubdomains = false, stripBoilerplate = true, boilerplateThreshold, hooks: hooksConfig, onPage } = options;
295
421
  const logger = resolveLogger(options);
296
422
  const hooks = createHooks();
297
423
  if (hooksConfig) hooks.addHooks(hooksConfig);
@@ -332,6 +458,7 @@ async function crawlAndGenerate(options, onProgress) {
332
458
  if (startingUrls.length > 0 && !skipSitemap && !singlePageMode) {
333
459
  const baseUrl = new URL(startingUrls[0]).origin;
334
460
  const homePageUrl = baseUrl;
461
+ const hasExplicitSitemaps = !!explicitSitemapUrls && explicitSitemapUrls.length > 0;
335
462
  onProgress?.(progress);
336
463
  let robotsContent = null;
337
464
  try {
@@ -349,90 +476,58 @@ async function crawlAndGenerate(options, onProgress) {
349
476
  logger.info(`[ROBOTS] Crawl-delay: ${crawlDelay}s`);
350
477
  }
351
478
  }
352
- if (robotsContent) {
353
- const sitemapMatches = robotsContent.match(ROBOTS_SITEMAP_RE);
354
- if (sitemapMatches && sitemapMatches.length > 0) {
355
- progress.sitemap.found = sitemapMatches.length;
356
- progress.sitemap.status = "processing";
357
- onProgress?.(progress);
358
- const robotsSitemaps = sitemapMatches.map((match) => match.replace(ROBOTS_SITEMAP_PREFIX_RE, "").trim());
359
- for (const sitemapUrl of robotsSitemaps) try {
360
- const robotsUrls = await loadSitemap(sitemapUrl);
361
- sitemapAttempts.push({
362
- url: sitemapUrl,
363
- success: true
364
- });
365
- const filteredUrls = filterSitemapUrls(robotsUrls, hasGlobPatterns, exclude, patterns, allowSubdomains);
366
- if (hasGlobPatterns) {
367
- startingUrls = filteredUrls;
368
- progress.sitemap.processed = filteredUrls.length;
369
- onProgress?.(progress);
370
- break;
371
- } else if (filteredUrls.length > 0) {
372
- startingUrls = filteredUrls;
373
- progress.sitemap.processed = filteredUrls.length;
374
- onProgress?.(progress);
375
- break;
376
- }
377
- } catch (error) {
378
- sitemapAttempts.push({
379
- url: sitemapUrl,
380
- success: false,
381
- error: error instanceof Error ? error.message : "Unknown error"
382
- });
383
- }
479
+ const candidateSitemaps = [];
480
+ if (hasExplicitSitemaps) candidateSitemaps.push(...explicitSitemapUrls);
481
+ else {
482
+ if (robotsContent) {
483
+ const sitemapMatches = robotsContent.match(ROBOTS_SITEMAP_RE);
484
+ if (sitemapMatches) for (const m of sitemapMatches) candidateSitemaps.push(m.replace(ROBOTS_SITEMAP_PREFIX_RE, "").trim());
384
485
  }
486
+ candidateSitemaps.push(`${baseUrl}/sitemap.xml`, `${baseUrl}/sitemap_index.xml`, `${baseUrl}/sitemaps.xml`, `${baseUrl}/sitemap-index.xml`);
385
487
  }
386
- let mainSitemapProcessed = false;
387
- const mainSitemapUrl = `${baseUrl}/sitemap.xml`;
388
- try {
389
- const sitemapUrls = await loadSitemap(mainSitemapUrl);
390
- sitemapAttempts.push({
391
- url: mainSitemapUrl,
392
- success: true
393
- });
394
- const filteredUrls = filterSitemapUrls(sitemapUrls, hasGlobPatterns, exclude, patterns, allowSubdomains);
395
- if (hasGlobPatterns) {
396
- startingUrls = filteredUrls;
397
- progress.sitemap.found = sitemapUrls.length;
398
- progress.sitemap.processed = filteredUrls.length;
399
- onProgress?.(progress);
400
- mainSitemapProcessed = true;
401
- } else if (filteredUrls.length > 0) {
488
+ if (candidateSitemaps.length > 0) {
489
+ progress.sitemap.status = "processing";
490
+ progress.sitemap.found = candidateSitemaps.length;
491
+ onProgress?.(progress);
492
+ }
493
+ if (hasExplicitSitemaps) {
494
+ const merged = [];
495
+ for (const sitemapUrl of candidateSitemaps) try {
496
+ merged.push(...await loadSitemap(sitemapUrl));
497
+ sitemapAttempts.push({
498
+ url: sitemapUrl,
499
+ success: true
500
+ });
501
+ } catch (error) {
502
+ sitemapAttempts.push({
503
+ url: sitemapUrl,
504
+ success: false,
505
+ error: error instanceof Error ? error.message : "Unknown error"
506
+ });
507
+ }
508
+ const dedupedUrls = merged.length > 0 ? [...new Set(merged)] : merged;
509
+ const filteredUrls = filterSitemapUrls(dedupedUrls, hasGlobPatterns, exclude, patterns, allowSubdomains);
510
+ if (hasGlobPatterns || filteredUrls.length > 0) {
402
511
  startingUrls = filteredUrls;
403
- progress.sitemap.found = sitemapUrls.length;
512
+ progress.sitemap.found = dedupedUrls.length;
404
513
  progress.sitemap.processed = filteredUrls.length;
405
514
  onProgress?.(progress);
406
- mainSitemapProcessed = true;
407
515
  }
408
- } catch (error) {
409
- sitemapAttempts.push({
410
- url: mainSitemapUrl,
411
- success: false,
412
- error: error instanceof Error ? error.message : "Unknown error"
413
- });
414
- if (!mainSitemapProcessed) {
415
- const commonSitemaps = [
416
- `${baseUrl}/sitemap_index.xml`,
417
- `${baseUrl}/sitemaps.xml`,
418
- `${baseUrl}/sitemap-index.xml`
419
- ];
420
- for (const sitemapUrl of commonSitemaps) try {
421
- const altUrls = await loadSitemap(sitemapUrl);
516
+ } else {
517
+ const seenCandidates = /* @__PURE__ */ new Set();
518
+ for (const sitemapUrl of candidateSitemaps) {
519
+ if (seenCandidates.has(sitemapUrl)) continue;
520
+ seenCandidates.add(sitemapUrl);
521
+ try {
522
+ const foundUrls = await loadSitemap(sitemapUrl);
422
523
  sitemapAttempts.push({
423
524
  url: sitemapUrl,
424
525
  success: true
425
526
  });
426
- const filteredUrls = filterSitemapUrls(altUrls, hasGlobPatterns, exclude, patterns, allowSubdomains);
427
- if (hasGlobPatterns) {
428
- startingUrls = filteredUrls;
429
- progress.sitemap.found = altUrls.length;
430
- progress.sitemap.processed = filteredUrls.length;
431
- onProgress?.(progress);
432
- break;
433
- } else if (filteredUrls.length > 0) {
527
+ const filteredUrls = filterSitemapUrls(foundUrls, hasGlobPatterns, exclude, patterns, allowSubdomains);
528
+ if (hasGlobPatterns || filteredUrls.length > 0) {
434
529
  startingUrls = filteredUrls;
435
- progress.sitemap.found = altUrls.length;
530
+ progress.sitemap.found = foundUrls.length;
436
531
  progress.sitemap.processed = filteredUrls.length;
437
532
  onProgress?.(progress);
438
533
  break;
@@ -448,6 +543,7 @@ async function crawlAndGenerate(options, onProgress) {
448
543
  }
449
544
  const successfulSitemaps = sitemapAttempts.filter((a) => a.success);
450
545
  const failedSitemaps = sitemapAttempts.filter((a) => !a.success);
546
+ if (hasExplicitSitemaps && failedSitemaps.length > 0) logger.warn(`Sitemap: ${failedSitemaps.length} explicit sitemap(s) failed to load; crawl may be incomplete: ${failedSitemaps.map((a) => `${a.url} (${a.error})`).join(", ")}`);
451
547
  if (successfulSitemaps.length > 0 && progress.sitemap.processed > 0) {
452
548
  sitemapProvidedUrls = true;
453
549
  logger.info(`Sitemap: ${progress.sitemap.processed} URLs from ${successfulSitemaps[0].url}`);
@@ -553,22 +649,8 @@ async function crawlAndGenerate(options, onProgress) {
553
649
  const hostPrefix = allowSubdomains ? [parsedUrl.hostname.replace(URL_PATH_UNSAFE_CHARS_RE, "-")] : [];
554
650
  const pathSegments = urlPath.replace(URL_TRAILING_SLASH_RE, "").split("/").filter((seg) => seg.length > 0);
555
651
  const safeSegments = [...hostPrefix, ...pathSegments.map((seg) => seg.replace(URL_PATH_UNSAFE_CHARS_RE, "-"))];
556
- filePath = join(outputDir, normalize(`${safeSegments.length > 0 ? safeSegments.join("/") : "index"}.md`));
557
- const contentCtx = {
558
- url,
559
- title,
560
- content: md,
561
- filePath
562
- };
563
- await hooks.callHook("crawl:content", contentCtx);
564
- md = contentCtx.content;
565
- filePath = contentCtx.filePath;
566
- const fileDir = dirname(filePath);
567
- if (fileDir && !createdDirs.has(fileDir)) {
568
- await mkdir(fileDir, { recursive: true });
569
- createdDirs.add(fileDir);
570
- }
571
- await writeFile(filePath, md, "utf-8");
652
+ const safeFilename = normalize(`${safeSegments.length > 0 ? safeSegments.join("/") : "index"}.md`);
653
+ filePath = join(outputDir, safeFilename);
572
654
  }
573
655
  const isHomePage = parsedUrl.pathname === "/" && parsedUrl.origin === normalizedHomePageUrl;
574
656
  if (shouldProcessMarkdown || isHomePage) {
@@ -763,6 +845,28 @@ async function crawlAndGenerate(options, onProgress) {
763
845
  progress.generation.status = "generating";
764
846
  onProgress?.(progress);
765
847
  const successfulResults = results.filter((r) => r.success);
848
+ if (stripBoilerplate && !singlePageMode) {
849
+ const cleaned = stripBoilerplateFromCorpus(successfulResults.map((r) => r.content), boilerplateThreshold !== void 0 ? { threshold: boilerplateThreshold } : {});
850
+ for (let i = 0; i < successfulResults.length; i++) successfulResults[i].content = cleaned[i];
851
+ }
852
+ if (generateIndividualMd) for (const result of successfulResults) {
853
+ if (!result.filePath) continue;
854
+ const contentCtx = {
855
+ url: result.url,
856
+ title: result.title,
857
+ content: result.content,
858
+ filePath: result.filePath
859
+ };
860
+ await hooks.callHook("crawl:content", contentCtx);
861
+ result.content = contentCtx.content;
862
+ result.filePath = contentCtx.filePath;
863
+ const fileDir = dirname(result.filePath);
864
+ if (fileDir && !createdDirs.has(fileDir)) {
865
+ await mkdir(fileDir, { recursive: true });
866
+ createdDirs.add(fileDir);
867
+ }
868
+ await writeFile(result.filePath, result.content, "utf-8");
869
+ }
766
870
  const firstUrl = new URL(withHttps(urls[0]));
767
871
  const originUrl = firstUrl.origin;
768
872
  const homePageResult = successfulResults.find((r) => {
@@ -813,4 +917,4 @@ async function crawlAndGenerate(options, onProgress) {
813
917
  }
814
918
  return results;
815
919
  }
816
- export { crawlAndGenerate, createClackLogger, parseUrlPattern, resolveLogger, silentLogger, validateGlobPattern };
920
+ export { DEFAULT_BOILERPLATE_THRESHOLD, crawlAndGenerate, createClackLogger, parseUrlPattern, resolveLogger, silentLogger, stripBoilerplateFromCorpus, validateGlobPattern };
package/dist/cli.mjs CHANGED
@@ -1,4 +1,4 @@
1
- import { crawlAndGenerate, parseUrlPattern, resolveLogger, validateGlobPattern } from "./_chunks/crawl.mjs";
1
+ import { DEFAULT_BOILERPLATE_THRESHOLD, crawlAndGenerate, parseUrlPattern, resolveLogger, validateGlobPattern } from "./_chunks/crawl.mjs";
2
2
  import { accessSync, constants, mkdirSync, readFileSync, unlinkSync, writeFileSync } from "node:fs";
3
3
  import { dirname, join, relative, resolve } from "pathe";
4
4
  import { withHttps } from "ufo";
@@ -12,7 +12,7 @@ async function loadMdreamConfig(cwd) {
12
12
  });
13
13
  return config || {};
14
14
  }
15
- const QUIET_FLAGS = new Set([
15
+ const QUIET_FLAGS = /* @__PURE__ */ new Set([
16
16
  "--silent",
17
17
  "--quiet",
18
18
  "-q"
@@ -224,7 +224,10 @@ Options:
224
224
  --crawl-delay <seconds> Crawl delay in seconds
225
225
  --exclude <pattern> Exclude URLs matching glob patterns (can be used multiple times)
226
226
  --skip-sitemap Skip sitemap.xml and robots.txt discovery
227
+ --sitemap <url> Use an explicit sitemap URL instead of auto-discovery (repeatable for multi-part sitemaps)
227
228
  --allow-subdomains Crawl across subdomains of the same root domain
229
+ --keep-boilerplate Keep repeated site chrome (nav/footer) in per-page output (default: stripped)
230
+ --boilerplate-threshold <n> Fraction of pages a block must repeat in to count as chrome (0-1, default: ${DEFAULT_BOILERPLATE_THRESHOLD})
228
231
  -v, --verbose Enable verbose logging
229
232
  -q, --quiet, --silent Suppress all logs (clean stdout for JSON-RPC/MCP)
230
233
  -h, --help Show this help message
@@ -238,6 +241,7 @@ Examples:
238
241
  @mdream/crawl -u example.com --exclude "*/admin/*" --exclude "*/api/*"
239
242
  @mdream/crawl -u example.com --verbose
240
243
  @mdream/crawl -u example.com --skip-sitemap
244
+ @mdream/crawl -u example.com --sitemap https://example.com/custom/sitemap.xml
241
245
  @mdream/crawl -u example.com --driver playwright --single-page
242
246
  `);
243
247
  process.exit(0);
@@ -349,7 +353,28 @@ Examples:
349
353
  const patterns = [parsed];
350
354
  const verbose = args.includes("--verbose") || args.includes("-v");
351
355
  const skipSitemap = args.includes("--skip-sitemap");
356
+ const sitemapUrls = getArgValues("--sitemap").map((u) => withHttps(u));
357
+ if (args.filter((a) => a === "--sitemap" || a === "-sitemap").length > sitemapUrls.length) {
358
+ logger.error("Error: --sitemap requires a URL value");
359
+ process.exit(1);
360
+ }
361
+ for (const sitemapUrl of sitemapUrls) try {
362
+ new URL(sitemapUrl);
363
+ } catch {
364
+ logger.error(`Error: Invalid sitemap URL: ${sitemapUrl}`);
365
+ process.exit(1);
366
+ }
352
367
  const allowSubdomains = args.includes("--allow-subdomains");
368
+ const stripBoilerplate = args.includes("--keep-boilerplate") ? false : void 0;
369
+ const boilerplateThresholdStr = getArgValue("--boilerplate-threshold");
370
+ let boilerplateThreshold;
371
+ if (boilerplateThresholdStr !== void 0) {
372
+ boilerplateThreshold = Number(boilerplateThresholdStr);
373
+ if (Number.isNaN(boilerplateThreshold) || boilerplateThreshold <= 0 || boilerplateThreshold > 1) {
374
+ logger.error("Error: Boilerplate threshold must be a number between 0 (exclusive) and 1");
375
+ process.exit(1);
376
+ }
377
+ }
353
378
  if (skipSitemap && parsed.isGlob) logger.warn("Warning: Using --skip-sitemap with glob URLs may not discover all matching pages.");
354
379
  return {
355
380
  urls: [url],
@@ -369,7 +394,10 @@ Examples:
369
394
  exclude: excludePatterns.length > 0 ? excludePatterns : void 0,
370
395
  verbose,
371
396
  skipSitemap,
397
+ sitemapUrls: sitemapUrls.length > 0 ? sitemapUrls : void 0,
372
398
  allowSubdomains,
399
+ stripBoilerplate,
400
+ boilerplateThreshold,
373
401
  silent
374
402
  };
375
403
  }
@@ -389,7 +417,10 @@ async function main() {
389
417
  maxDepth: cliOptions.maxDepth ?? fileConfig.maxDepth,
390
418
  crawlDelay: cliOptions.crawlDelay ?? fileConfig.crawlDelay,
391
419
  skipSitemap: cliOptions.skipSitemap || fileConfig.skipSitemap || false,
420
+ sitemapUrls: cliOptions.sitemapUrls ?? (fileConfig.sitemap ? Array.isArray(fileConfig.sitemap) ? fileConfig.sitemap : [fileConfig.sitemap] : void 0),
392
421
  allowSubdomains: cliOptions.allowSubdomains || fileConfig.allowSubdomains || false,
422
+ stripBoilerplate: cliOptions.stripBoilerplate ?? fileConfig.stripBoilerplate ?? true,
423
+ boilerplateThreshold: cliOptions.boilerplateThreshold ?? fileConfig.boilerplateThreshold,
393
424
  verbose: cliOptions.verbose || fileConfig.verbose || false,
394
425
  silent,
395
426
  exclude: configExclude.length > 0 || cliExclude.length > 0 ? [...configExclude, ...cliExclude] : void 0,
@@ -407,6 +438,7 @@ async function main() {
407
438
  `Formats: ${formats.join(", ")}`,
408
439
  options.exclude && options.exclude.length > 0 && `Exclude: ${options.exclude.join(", ")}`,
409
440
  options.skipSitemap && `Skip sitemap: Yes`,
441
+ options.sitemapUrls && options.sitemapUrls.length > 0 && `Sitemap: ${options.sitemapUrls.join(", ")}`,
410
442
  options.allowSubdomains && `Allow subdomains: Yes`,
411
443
  options.verbose && `Verbose: Enabled`
412
444
  ].filter(Boolean);
package/dist/index.d.mts CHANGED
@@ -1,3 +1,34 @@
1
+ interface BoilerplateOptions {
2
+ /**
3
+ * Fraction of documents a shingle must appear in to count as chrome (0..1).
4
+ * Higher = stricter. Defaults to `DEFAULT_BOILERPLATE_THRESHOLD` (0.5).
5
+ */
6
+ threshold?: number;
7
+ /**
8
+ * Minimum number of documents before detection runs at all.
9
+ * Defaults to `DEFAULT_BOILERPLATE_MIN_DOCS` (3).
10
+ */
11
+ minDocs?: number;
12
+ /**
13
+ * Number of consecutive tokens per shingle. Smaller catches shorter repeated
14
+ * spans but risks matching common phrases; larger is more conservative.
15
+ * Defaults to `DEFAULT_BOILERPLATE_SHINGLE_SIZE` (6).
16
+ */
17
+ shingleSize?: number;
18
+ /**
19
+ * Minimum length (in tokens) of a covered run before it is removed. Stops short
20
+ * incidental matches inside unique prose from being nibbled away.
21
+ * Defaults to `DEFAULT_BOILERPLATE_MIN_RUN` (12).
22
+ */
23
+ minRun?: number;
24
+ }
25
+ /**
26
+ * Strip repeated cross-page boilerplate from a corpus of markdown documents.
27
+ *
28
+ * Returns a new array aligned with `contents`. Documents that are unchanged (no
29
+ * detected chrome, or detection skipped) are returned byte-for-byte as given.
30
+ */
31
+ declare function stripBoilerplateFromCorpus(contents: string[], options?: BoilerplateOptions): string[];
1
32
  /**
2
33
  * Minimal spinner surface used by the crawler. Mirrors the subset of
3
34
  * `@clack/prompts` spinner methods the package relies on.
@@ -95,7 +126,28 @@ interface CrawlOptions {
95
126
  descriptionOverride?: string;
96
127
  verbose?: boolean;
97
128
  skipSitemap?: boolean;
129
+ /**
130
+ * Explicit sitemap URL(s) to use instead of auto-discovery (robots.txt +
131
+ * well-known paths). Accepts non-standard locations and multiple parts, which
132
+ * are all loaded and merged. Ignored when `skipSitemap` is set.
133
+ */
134
+ sitemapUrls?: string[];
98
135
  allowSubdomains?: boolean;
136
+ /**
137
+ * Strip repeated site chrome (top nav, footer, newsletter walls) from per-page
138
+ * markdown and llms-full.txt page sections. Implemented as a post-processing
139
+ * pass over the generated markdown: blocks that repeat across the corpus are
140
+ * detected by frequency and removed only from the wrapping (leading/trailing)
141
+ * run of each page, so a page's main content is preserved. Does not affect the
142
+ * llms.txt link index. Defaults to true. Needs a corpus (>= 3 pages) to run, so
143
+ * single-page crawls are left unchanged.
144
+ */
145
+ stripBoilerplate?: boolean;
146
+ /**
147
+ * Fraction of crawled pages a block must appear in to count as chrome (0..1).
148
+ * Higher is stricter. Defaults to `DEFAULT_BOILERPLATE_THRESHOLD` (0.5).
149
+ */
150
+ boilerplateThreshold?: number;
99
151
  /**
100
152
  * Suppress all diagnostic/progress logging. Use when stdout must stay clean,
101
153
  * e.g. an MCP server that only emits JSON-RPC (issue #100). Ignored when an
@@ -107,7 +159,7 @@ interface CrawlOptions {
107
159
  * stderr) instead of the default `@clack/prompts` stdout output.
108
160
  */
109
161
  logger?: CrawlLogger;
110
- hooks?: Partial<{ [K in keyof CrawlHooks]: CrawlHooks[K] | CrawlHooks[K][] }>;
162
+ hooks?: Partial<{ [K in keyof CrawlHooks]: CrawlHooks[K] | CrawlHooks[K][]; }>;
111
163
  onPage?: (page: PageData) => Promise<void> | void;
112
164
  }
113
165
  interface MdreamCrawlConfig {
@@ -117,12 +169,18 @@ interface MdreamCrawlConfig {
117
169
  maxPages?: number;
118
170
  crawlDelay?: number;
119
171
  skipSitemap?: boolean;
172
+ /** Explicit sitemap URL(s) to use instead of auto-discovery. Supports non-standard locations and multiple parts. */
173
+ sitemap?: string | string[];
120
174
  allowSubdomains?: boolean;
175
+ /** Strip repeated site chrome from per-page output. Defaults to true. */
176
+ stripBoilerplate?: boolean;
177
+ /** Fraction of pages a block must repeat in to count as chrome (0..1). Defaults to 0.5. */
178
+ boilerplateThreshold?: number;
121
179
  verbose?: boolean;
122
180
  /** Suppress all diagnostic/progress logging (issue #100). */
123
181
  silent?: boolean;
124
182
  artifacts?: ('llms.txt' | 'llms-full.txt' | 'markdown')[];
125
- hooks?: Partial<{ [K in keyof CrawlHooks]: CrawlHooks[K] | CrawlHooks[K][] }>;
183
+ hooks?: Partial<{ [K in keyof CrawlHooks]: CrawlHooks[K] | CrawlHooks[K][]; }>;
126
184
  }
127
185
  declare function defineConfig(config: MdreamCrawlConfig): MdreamCrawlConfig;
128
186
  interface ParsedUrlPattern {
@@ -159,7 +217,8 @@ interface CrawlProgress {
159
217
  total: number;
160
218
  processed: number;
161
219
  failed: number;
162
- currentUrl?: string; /** Page fetch latency stats in ms */
220
+ currentUrl?: string;
221
+ /** Page fetch latency stats in ms */
163
222
  latency: {
164
223
  total: number;
165
224
  min: number;
@@ -173,4 +232,4 @@ interface CrawlProgress {
173
232
  };
174
233
  }
175
234
  declare function crawlAndGenerate(options: CrawlOptions, onProgress?: (progress: CrawlProgress) => void): Promise<CrawlResult[]>;
176
- export { type CrawlHooks, type CrawlLogger, type CrawlOptions, type CrawlResult, type CrawlSpinner, type MdreamCrawlConfig, type PageData, crawlAndGenerate, createClackLogger, defineConfig, resolveLogger, silentLogger };
235
+ export { type BoilerplateOptions, type CrawlHooks, type CrawlLogger, type CrawlOptions, type CrawlResult, type CrawlSpinner, type MdreamCrawlConfig, type PageData, crawlAndGenerate, createClackLogger, defineConfig, resolveLogger, silentLogger, stripBoilerplateFromCorpus };
package/dist/index.mjs CHANGED
@@ -1,5 +1,5 @@
1
- import { crawlAndGenerate, createClackLogger, resolveLogger, silentLogger } from "./_chunks/crawl.mjs";
1
+ import { crawlAndGenerate, createClackLogger, resolveLogger, silentLogger, stripBoilerplateFromCorpus } from "./_chunks/crawl.mjs";
2
2
  function defineConfig(config) {
3
3
  return config;
4
4
  }
5
- export { crawlAndGenerate, createClackLogger, defineConfig, resolveLogger, silentLogger };
5
+ export { crawlAndGenerate, createClackLogger, defineConfig, resolveLogger, silentLogger, stripBoilerplateFromCorpus };
package/package.json CHANGED
@@ -1,7 +1,7 @@
1
1
  {
2
2
  "name": "@mdream/crawl",
3
3
  "type": "module",
4
- "version": "1.4.1",
4
+ "version": "1.5.2",
5
5
  "description": "Mdream Crawl generates comprehensive llms.txt artifacts from a single URL, using mdream to convert HTML to Markdown.",
6
6
  "author": {
7
7
  "name": "Harlan Wilton",
@@ -54,17 +54,17 @@
54
54
  }
55
55
  },
56
56
  "dependencies": {
57
- "@clack/prompts": "^1.5.1",
57
+ "@clack/prompts": "^1.7.0",
58
58
  "c12": "^3.3.4",
59
59
  "hookable": "^6.1.1",
60
- "nypm": "^0.6.7",
60
+ "nypm": "^0.6.8",
61
61
  "ofetch": "^1.5.1",
62
62
  "pathe": "^2.0.3",
63
- "picomatch": "^4.0.4",
64
- "tldts": "^7.4.2",
63
+ "picomatch": "^4.0.5",
64
+ "tldts": "^7.4.8",
65
65
  "ufo": "^1.6.4",
66
- "@mdream/js": "1.4.1",
67
- "mdream": "1.4.1"
66
+ "@mdream/js": "1.5.2",
67
+ "mdream": "1.5.2"
68
68
  },
69
69
  "devDependencies": {
70
70
  "@types/picomatch": "^4.0.3"