@mdream/crawl 1.0.2 → 1.0.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -9,24 +9,14 @@ import { dirname, join, normalize, resolve } from "pathe";
9
9
  import { withHttps } from "ufo";
10
10
  import picomatch from "picomatch";
11
11
  import { getDomain } from "tldts";
12
- //#region src/glob-utils.ts
13
12
  function stripGlobTail(s) {
14
13
  const idx = s.indexOf("*");
15
14
  return idx === -1 ? s : s.slice(0, idx);
16
15
  }
17
16
  const GLOB_CHAR_RE = /[*?[]/;
18
- /**
19
- * Extract the registrable domain from a hostname using the public suffix list.
20
- * Handles multi-part TLDs (.co.uk, .github.io, etc.) correctly.
21
- * Returns the hostname unchanged for IPs or when parsing fails.
22
- */
23
17
  function getRegistrableDomain(hostname) {
24
18
  return getDomain(hostname, { allowPrivateDomains: true }) || hostname;
25
19
  }
26
- /**
27
- * Parse a URL that may contain glob patterns
28
- * Example: https://nuxtseo.com/docs/** -> { baseUrl: "https://nuxtseo.com", pattern: "/docs/**", isGlob: true }
29
- */
30
20
  function parseUrlPattern(input) {
31
21
  if (!(input.includes("*") || input.includes("?") || input.includes("["))) return {
32
22
  baseUrl: input,
@@ -47,9 +37,6 @@ function parseUrlPattern(input) {
47
37
  throw new Error(`Invalid URL pattern: "${input}". Please provide a valid URL with glob patterns (e.g., "example.com/docs/*" or "https://example.com/api/**").`);
48
38
  }
49
39
  }
50
- /**
51
- * Check if a URL matches a glob pattern
52
- */
53
40
  function matchesGlobPattern(url, parsedPattern, allowSubdomains = false) {
54
41
  if (!parsedPattern.isGlob) return true;
55
42
  try {
@@ -69,10 +56,6 @@ function matchesGlobPattern(url, parsedPattern, allowSubdomains = false) {
69
56
  return false;
70
57
  }
71
58
  }
72
- /**
73
- * Get the starting URL for crawling from a glob pattern
74
- * For https://nuxtseo.com/docs/**, we want to start at https://nuxtseo.com
75
- */
76
59
  function getStartingUrl(parsedPattern) {
77
60
  if (!parsedPattern.isGlob) return withHttps(parsedPattern.baseUrl);
78
61
  const pattern = parsedPattern.pattern;
@@ -83,9 +66,6 @@ function getStartingUrl(parsedPattern) {
83
66
  const pathBeforeGlob = lastSlash >= 0 ? beforeGlob.substring(0, lastSlash + 1) : "/";
84
67
  return withHttps(parsedPattern.baseUrl + pathBeforeGlob);
85
68
  }
86
- /**
87
- * Check if a URL should be excluded based on exclude patterns
88
- */
89
69
  function isUrlExcluded(url, excludePatterns, allowSubdomains = false) {
90
70
  if (!excludePatterns || excludePatterns.length === 0) return false;
91
71
  try {
@@ -104,15 +84,9 @@ function isUrlExcluded(url, excludePatterns, allowSubdomains = false) {
104
84
  return false;
105
85
  }
106
86
  }
107
- /**
108
- * Check if a string is valid sitemap XML content (not an HTML page or other non-sitemap response)
109
- */
110
87
  function isValidSitemapXml(content) {
111
88
  return content.includes("<urlset") || content.includes("<sitemapindex");
112
89
  }
113
- /**
114
- * Validate glob pattern syntax
115
- */
116
90
  function validateGlobPattern(pattern) {
117
91
  try {
118
92
  parseUrlPattern(pattern);
@@ -121,8 +95,6 @@ function validateGlobPattern(pattern) {
121
95
  return `Invalid glob pattern: ${error instanceof Error ? error.message : error}`;
122
96
  }
123
97
  }
124
- //#endregion
125
- //#region src/crawl.ts
126
98
  const SITEMAP_INDEX_LOC_RE = /<sitemap[^>]*>.*?<loc>(.*?)<\/loc>.*?<\/sitemap>/gs;
127
99
  const SITEMAP_URL_LOC_RE = /<url[^>]*>.*?<loc>(.*?)<\/loc>.*?<\/url>/gs;
128
100
  const ROBOTS_SITEMAP_RE = /Sitemap:\s*(.*)/gi;
@@ -131,15 +103,70 @@ const ROBOTS_CRAWL_DELAY_RE = /Crawl-delay:\s*(\d+(?:\.\d+)?)/i;
131
103
  const URL_TRAILING_SLASH_RE = /\/$/;
132
104
  const URL_PATH_UNSAFE_CHARS_RE = /[^\w\-]/g;
133
105
  const FRONTMATTER_BLOCK_RE = /^---[^\n]*\n[\s\S]*?\n---[^\n]*\n?/;
106
+ const TRACKING_PARAMS = new Set([
107
+ "utm_source",
108
+ "utm_medium",
109
+ "utm_campaign",
110
+ "utm_term",
111
+ "utm_content",
112
+ "utm_id",
113
+ "fbclid",
114
+ "gclid",
115
+ "gclsrc",
116
+ "dclid",
117
+ "gbraid",
118
+ "wbraid",
119
+ "msclkid",
120
+ "twclid",
121
+ "li_fat_id",
122
+ "mc_cid",
123
+ "mc_eid",
124
+ "ref",
125
+ "source",
126
+ "sessionid",
127
+ "session_id",
128
+ "sid",
129
+ "_ga",
130
+ "_gl",
131
+ "_hsenc",
132
+ "_hsmi",
133
+ "_openstat",
134
+ "yclid",
135
+ "ymclid",
136
+ "spm",
137
+ "scm"
138
+ ]);
139
+ const MAX_QUERY_VARIANTS_PER_PATH = 5;
134
140
  const FETCH_HEADERS = {
135
141
  "User-Agent": "mdream-crawler/1.0",
136
142
  "Accept": "text/html,application/xhtml+xml,text/markdown"
137
143
  };
138
144
  const DEFAULT_CONCURRENCY = 20;
145
+ const IGNORED_PATH_PREFIXES = [
146
+ "/cdn-cgi/",
147
+ "/_next/",
148
+ "/_nuxt/",
149
+ "/__",
150
+ "/wp-admin/",
151
+ "/wp-json/",
152
+ "/wp-includes/",
153
+ "/wp-content/uploads/",
154
+ "/api/",
155
+ "/assets/",
156
+ "/static/"
157
+ ];
158
+ const HTML_EXTENSIONS_RE = /\.(?:html?|php|aspx?|jsp)$/i;
139
159
  function extractCdataUrl(url) {
140
160
  if (url.startsWith("<![CDATA[") && url.endsWith("]]>")) return url.slice(9, -3);
141
161
  return url;
142
162
  }
163
+ function normalizeUrl(resolved) {
164
+ resolved.hash = "";
165
+ const params = resolved.searchParams;
166
+ for (const key of [...params.keys()]) if (TRACKING_PARAMS.has(key)) params.delete(key);
167
+ params.sort();
168
+ return resolved.href.replace(URL_TRAILING_SLASH_RE, "") || resolved.href;
169
+ }
143
170
  async function loadSitemap(sitemapUrl) {
144
171
  const xmlContent = await ofetch(sitemapUrl, {
145
172
  headers: FETCH_HEADERS,
@@ -186,7 +213,7 @@ function extractMetadataInline(parsedUrl, allowedDomains) {
186
213
  const href = el.attributes.href;
187
214
  if (href) try {
188
215
  const resolved = new URL(href, url);
189
- const absoluteUrl = resolved.href;
216
+ const absoluteUrl = normalizeUrl(resolved);
190
217
  if (allowedDomains) {
191
218
  const domain = getRegistrableDomain(resolved.hostname);
192
219
  if (domain && allowedDomains.has(domain)) links.add(absoluteUrl);
@@ -269,6 +296,7 @@ async function crawlAndGenerate(options, onProgress) {
269
296
  generation: { status: "idle" }
270
297
  };
271
298
  const sitemapAttempts = [];
299
+ let sitemapProvidedUrls = false;
272
300
  if (startingUrls.length > 0 && !skipSitemap && !singlePageMode) {
273
301
  const baseUrl = new URL(startingUrls[0]).origin;
274
302
  const homePageUrl = baseUrl;
@@ -388,15 +416,11 @@ async function crawlAndGenerate(options, onProgress) {
388
416
  }
389
417
  const successfulSitemaps = sitemapAttempts.filter((a) => a.success);
390
418
  const failedSitemaps = sitemapAttempts.filter((a) => !a.success);
391
- if (successfulSitemaps.length > 0) {
392
- const sitemapUrl = successfulSitemaps[0].url;
393
- if (progress.sitemap.processed > 0) p.note(`Found sitemap at ${sitemapUrl} with ${progress.sitemap.processed} URLs`, "Sitemap Discovery");
394
- else p.note(`Found sitemap at ${sitemapUrl} but no URLs matched your search criteria`, "Sitemap Discovery");
395
- } else if (failedSitemaps.length > 0) {
396
- const firstAttempt = failedSitemaps[0];
397
- if (firstAttempt.error?.includes("404")) p.note(`No sitemap found, using crawler to discover pages`, "Sitemap Discovery");
398
- else p.note(`Could not access sitemap: ${firstAttempt.error}`, "Sitemap Discovery");
399
- }
419
+ if (successfulSitemaps.length > 0 && progress.sitemap.processed > 0) {
420
+ sitemapProvidedUrls = true;
421
+ p.log.info(`Sitemap: ${progress.sitemap.processed} URLs from ${successfulSitemaps[0].url}`);
422
+ } else if (successfulSitemaps.length > 0) p.log.info(`Sitemap: found but no URLs matched filters`);
423
+ else if (failedSitemaps.length > 0) p.log.info(`Sitemap: not found, discovering pages via crawler`);
400
424
  if (!startingUrls.includes(homePageUrl)) startingUrls.unshift(homePageUrl);
401
425
  progress.sitemap.status = "completed";
402
426
  progress.crawling.total = startingUrls.length;
@@ -418,6 +442,16 @@ async function crawlAndGenerate(options, onProgress) {
418
442
  return "";
419
443
  }
420
444
  }).filter(Boolean)) : void 0;
445
+ const isContentUrl = (url) => {
446
+ try {
447
+ const pathname = new URL(url).pathname;
448
+ for (let i = 0; i < IGNORED_PATH_PREFIXES.length; i++) if (pathname.startsWith(IGNORED_PATH_PREFIXES[i])) return false;
449
+ if (pathname.lastIndexOf(".") > pathname.lastIndexOf("/")) return HTML_EXTENSIONS_RE.test(pathname);
450
+ return true;
451
+ } catch {
452
+ return false;
453
+ }
454
+ };
421
455
  const shouldCrawlUrl = (url) => {
422
456
  if (isUrlExcluded(url, exclude, allowSubdomains)) return false;
423
457
  if (!hasGlobPatterns) {
@@ -440,6 +474,8 @@ async function crawlAndGenerate(options, onProgress) {
440
474
  const normalizedHomePageUrl = (startingUrls.length > 0 ? new URL(startingUrls[0]).origin : "").replace(URL_TRAILING_SLASH_RE, "");
441
475
  const createdDirs = /* @__PURE__ */ new Set();
442
476
  const sharedOrigin = origin || "";
477
+ const pendingUrls = [];
478
+ const queryVariantCounts = /* @__PURE__ */ new Map();
443
479
  const processPage = async (url, content, initialTitle, depth, isMarkdown = false) => {
444
480
  const parsedUrl = new URL(url);
445
481
  const shouldProcessMarkdown = shouldCrawlUrl(url);
@@ -511,12 +547,28 @@ async function crawlAndGenerate(options, onProgress) {
511
547
  progress.crawling.processed = results.length;
512
548
  onProgress?.(progress);
513
549
  }
514
- if (followLinks && !singlePageMode && depth < maxDepth) {
515
- const filteredLinks = metadata.links.filter((link) => shouldCrawlUrl(link));
516
- for (const link of filteredLinks) processedUrls.add(link);
550
+ if (followLinks && !singlePageMode && !sitemapProvidedUrls && depth < maxDepth) {
551
+ const filteredLinks = metadata.links.filter((link) => shouldCrawlUrl(link) && isContentUrl(link));
552
+ for (const link of filteredLinks) {
553
+ if (processedUrls.has(link)) continue;
554
+ const linkUrl = new URL(link);
555
+ if (linkUrl.search) {
556
+ const pathname = linkUrl.pathname;
557
+ const count = queryVariantCounts.get(pathname) || 0;
558
+ if (count >= MAX_QUERY_VARIANTS_PER_PATH) continue;
559
+ queryVariantCounts.set(pathname, count + 1);
560
+ }
561
+ processedUrls.add(link);
562
+ pendingUrls.push({
563
+ url: link,
564
+ depth: depth + 1
565
+ });
566
+ }
517
567
  }
518
568
  };
519
569
  const urlsToProcess = startingUrls.slice(0, maxRequestsPerCrawl);
570
+ for (const url of urlsToProcess) processedUrls.add(url.replace(URL_TRAILING_SLASH_RE, "") || url);
571
+ let totalProcessed = 0;
520
572
  progress.crawling.status = "processing";
521
573
  progress.crawling.total = urlsToProcess.length;
522
574
  onProgress?.(progress);
@@ -569,12 +621,24 @@ async function crawlAndGenerate(options, onProgress) {
569
621
  useChrome
570
622
  };
571
623
  const crawler = new PlaywrightCrawler(crawlerOptions);
572
- const initialRequests = urlsToProcess.map((url) => ({
624
+ const allRequests = [...urlsToProcess.map((url) => ({
573
625
  url,
574
626
  userData: { depth: 0 }
575
- }));
627
+ }))];
576
628
  try {
577
- await crawler.run(initialRequests);
629
+ await crawler.run(allRequests);
630
+ totalProcessed += urlsToProcess.length;
631
+ while (pendingUrls.length > 0 && totalProcessed < maxRequestsPerCrawl) {
632
+ const batch = pendingUrls.splice(0, maxRequestsPerCrawl - totalProcessed);
633
+ progress.crawling.total += batch.length;
634
+ onProgress?.(progress);
635
+ const batchRequests = batch.map((item) => ({
636
+ url: item.url,
637
+ userData: { depth: item.depth }
638
+ }));
639
+ await crawler.run(batchRequests);
640
+ totalProcessed += batch.length;
641
+ }
578
642
  } catch (error) {
579
643
  const msg = error instanceof Error ? error.message : "";
580
644
  if (msg.includes("wmic") || msg.includes("ENOENT")) throw new Error(`Crawlee failed to spawn a system process (${msg}). On Windows 11+, wmic.exe is no longer available. Upgrade crawlee to >=3.16.0 or use the HTTP driver instead (--driver http).`);
@@ -585,60 +649,74 @@ async function crawlAndGenerate(options, onProgress) {
585
649
  throw error;
586
650
  }
587
651
  await purgeDefaultStorages();
588
- } else await runConcurrent(urlsToProcess, DEFAULT_CONCURRENCY, async (url) => {
589
- progress.crawling.currentUrl = url;
590
- onProgress?.(progress);
591
- if (crawlDelay) {
592
- const delay = crawlDelay;
593
- await new Promise((resolve) => setTimeout(resolve, delay * 1e3));
594
- }
595
- const urlCtx = {
596
- url,
597
- skip: false
598
- };
599
- await hooks.callHook("crawl:url", urlCtx);
600
- if (urlCtx.skip) return;
601
- try {
602
- const fetchStart = Date.now();
603
- const response = await ofetch.raw(url, {
604
- headers: FETCH_HEADERS,
605
- responseType: "text",
606
- retry: 2,
607
- retryDelay: 500,
608
- timeout: 1e4,
609
- onResponseError({ response }) {
610
- if (response.status === 429) {
611
- const retryAfter = response.headers.get("retry-after");
612
- const delaySec = retryAfter ? Number.parseInt(retryAfter) || 1 : 2;
613
- if (!crawlDelay || delaySec > crawlDelay) crawlDelay = delaySec;
614
- }
615
- }
616
- });
617
- recordLatency(Date.now() - fetchStart);
618
- const body = response._data ?? "";
619
- const contentType = response.headers.get("content-type") || "";
620
- await processPage(url, body, "", 0, contentType.includes("text/markdown") || contentType.includes("text/x-markdown"));
621
- } catch (error) {
622
- if (verbose) console.error(`[ERROR] URL: ${url}, Error: ${error instanceof Error ? error.message : "Unknown"}`);
623
- progress.crawling.failed++;
624
- results.push({
652
+ } else {
653
+ const fetchPage = async (url, depth) => {
654
+ progress.crawling.currentUrl = url;
655
+ onProgress?.(progress);
656
+ if (crawlDelay) {
657
+ const delay = crawlDelay;
658
+ await new Promise((resolve) => setTimeout(resolve, delay * 1e3));
659
+ }
660
+ const urlCtx = {
625
661
  url,
626
- title: "",
627
- content: "",
628
- timestamp: Date.now(),
629
- success: false,
630
- error: error instanceof Error ? error.message : "Unknown error",
631
- metadata: {
662
+ skip: false
663
+ };
664
+ await hooks.callHook("crawl:url", urlCtx);
665
+ if (urlCtx.skip) return;
666
+ try {
667
+ const fetchStart = Date.now();
668
+ const response = await ofetch.raw(url, {
669
+ headers: FETCH_HEADERS,
670
+ responseType: "text",
671
+ retry: 2,
672
+ retryDelay: 500,
673
+ timeout: 1e4,
674
+ onResponseError({ response }) {
675
+ if (response.status === 429) {
676
+ const retryAfter = response.headers.get("retry-after");
677
+ const delaySec = retryAfter ? Number.parseInt(retryAfter) || 1 : 2;
678
+ if (!crawlDelay || delaySec > crawlDelay) crawlDelay = delaySec;
679
+ }
680
+ }
681
+ });
682
+ recordLatency(Date.now() - fetchStart);
683
+ const body = response._data ?? "";
684
+ const contentType = response.headers.get("content-type") || "";
685
+ const isHtml = contentType.includes("text/html") || contentType.includes("application/xhtml");
686
+ const isMarkdown = contentType.includes("text/markdown") || contentType.includes("text/x-markdown");
687
+ if (!isHtml && !isMarkdown) return;
688
+ await processPage(url, body, "", depth, isMarkdown);
689
+ } catch (error) {
690
+ if (verbose) console.error(`[ERROR] URL: ${url}, Error: ${error instanceof Error ? error.message : "Unknown"}`);
691
+ progress.crawling.failed++;
692
+ results.push({
693
+ url,
632
694
  title: "",
633
- description: "",
634
- links: []
635
- },
636
- depth: 0
637
- });
638
- progress.crawling.processed = results.length;
695
+ content: "",
696
+ timestamp: Date.now(),
697
+ success: false,
698
+ error: error instanceof Error ? error.message : "Unknown error",
699
+ metadata: {
700
+ title: "",
701
+ description: "",
702
+ links: []
703
+ },
704
+ depth
705
+ });
706
+ progress.crawling.processed = results.length;
707
+ onProgress?.(progress);
708
+ }
709
+ };
710
+ await runConcurrent(urlsToProcess, DEFAULT_CONCURRENCY, (url) => fetchPage(url, 0));
711
+ totalProcessed += urlsToProcess.length;
712
+ while (pendingUrls.length > 0 && totalProcessed < maxRequestsPerCrawl) {
713
+ const batch = pendingUrls.splice(0, maxRequestsPerCrawl - totalProcessed);
714
+ progress.crawling.total += batch.length;
639
715
  onProgress?.(progress);
716
+ await runConcurrent(batch, DEFAULT_CONCURRENCY, (item) => fetchPage(item.url, item.depth));
717
+ totalProcessed += batch.length;
640
718
  }
641
- });
719
+ }
642
720
  progress.crawling.status = "completed";
643
721
  onProgress?.(progress);
644
722
  await hooks.callHook("crawl:done", { results });
@@ -696,5 +774,4 @@ async function crawlAndGenerate(options, onProgress) {
696
774
  }
697
775
  return results;
698
776
  }
699
- //#endregion
700
777
  export { parseUrlPattern as n, validateGlobPattern as r, crawlAndGenerate as t };
@@ -1,6 +1,5 @@
1
1
  import * as p from "@clack/prompts";
2
2
  import { addDependency } from "nypm";
3
- //#region src/playwright-utils.ts
4
3
  async function checkPlaywrightInstallation() {
5
4
  try {
6
5
  await import("playwright");
@@ -55,5 +54,4 @@ async function isUseChromeSupported() {
55
54
  } catch {}
56
55
  return false;
57
56
  }
58
- //#endregion
59
57
  export { ensurePlaywrightInstalled, isUseChromeSupported };
package/dist/cli.mjs CHANGED
@@ -1,11 +1,10 @@
1
1
  import { n as parseUrlPattern, r as validateGlobPattern, t as crawlAndGenerate } from "./_chunks/crawl.mjs";
2
2
  import { accessSync, constants, mkdirSync, readFileSync, unlinkSync, writeFileSync } from "node:fs";
3
3
  import * as p from "@clack/prompts";
4
- import { dirname, join, resolve } from "pathe";
4
+ import { dirname, join, relative, resolve } from "pathe";
5
5
  import { withHttps } from "ufo";
6
6
  import { fileURLToPath } from "node:url";
7
7
  import { loadConfig } from "c12";
8
- //#region src/config.ts
9
8
  async function loadMdreamConfig(cwd) {
10
9
  const { config } = await loadConfig({
11
10
  name: "mdream",
@@ -13,8 +12,6 @@ async function loadMdreamConfig(cwd) {
13
12
  });
14
13
  return config || {};
15
14
  }
16
- //#endregion
17
- //#region src/cli.ts
18
15
  const packageJsonPath = join(dirname(fileURLToPath(import.meta.url)), "..", "package.json");
19
16
  const version = JSON.parse(readFileSync(packageJsonPath, "utf-8")).version;
20
17
  function checkOutputDirectoryPermissions(outputDir) {
@@ -152,7 +149,7 @@ async function interactiveCrawl() {
152
149
  });
153
150
  const summary = [
154
151
  `URLs: ${urls.join(", ")}`,
155
- `Output: ${outputDir}`,
152
+ `Output: ${relative(process.cwd(), outputDir) || outputDir}`,
156
153
  `Driver: ${crawlerOptions.driver}`,
157
154
  `Max pages: Unlimited`,
158
155
  `Follow links: Yes (depth 3)`,
@@ -179,19 +176,16 @@ async function interactiveCrawl() {
179
176
  };
180
177
  }
181
178
  async function showCrawlResults(successful, failed, outputDir, generatedFiles, durationSeconds, latency) {
182
- const messages = [];
183
- const durationStr = `${durationSeconds.toFixed(1)}s`;
184
- messages.push(`📄 ${successful} pages \u00B7 ⏱️ ${durationStr}`);
185
- if (failed > 0) messages.push(`⚠️ ${failed} failed`);
179
+ let line = `${successful} pages in ${`${durationSeconds.toFixed(1)}s`}`;
180
+ if (failed > 0) line += ` (${failed} failed)`;
186
181
  if (latency && latency.count > 0) {
187
182
  const avg = Math.round(latency.total / latency.count);
188
183
  const min = latency.min === Infinity ? 0 : Math.round(latency.min);
189
184
  const max = Math.round(latency.max);
190
- messages.push(`🏓 avg ${avg}ms \u00B7 min ${min}ms \u00B7 max ${max}ms`);
185
+ line += ` \u00B7 HTTP Latency: avg ${avg}ms, min ${min}ms, max ${max}ms`;
191
186
  }
192
- messages.push(`📦 ${generatedFiles.join(", ")}`);
193
- messages.push(`📁 ${outputDir}`);
194
- p.note(messages.join("\n"), "✅ Complete");
187
+ p.log.success(line);
188
+ p.log.info(`${generatedFiles.join(", ")} \u2192 ${relative(process.cwd(), outputDir) || "."}`);
195
189
  }
196
190
  function parseCliArgs() {
197
191
  const args = process.argv.slice(2);
@@ -391,9 +385,8 @@ async function main() {
391
385
  if (options.generateIndividualMd) formats.push("Individual MD files");
392
386
  const summary = [
393
387
  `URL: ${options.urls.join(", ")}`,
394
- `Output: ${options.outputDir}`,
395
- `Driver: ${options.driver}`,
396
- `Depth: ${options.maxDepth}`,
388
+ `Output: ${relative(process.cwd(), options.outputDir) || "."}`,
389
+ `Driver: ${options.driver} \u00B7 Depth: ${options.maxDepth}`,
397
390
  `Formats: ${formats.join(", ")}`,
398
391
  options.exclude && options.exclude.length > 0 && `Exclude: ${options.exclude.join(", ")}`,
399
392
  options.skipSitemap && `Skip sitemap: Yes`,
@@ -481,5 +474,4 @@ main().catch((error) => {
481
474
  else p.log.error(`Unexpected error: ${msg}`);
482
475
  process.exit(1);
483
476
  });
484
- //#endregion
485
477
  export {};
package/dist/index.d.mts CHANGED
@@ -1,4 +1,3 @@
1
- //#region src/types.d.ts
2
1
  interface PageData {
3
2
  url: string;
4
3
  html: string;
@@ -82,8 +81,6 @@ interface CrawlResult {
82
81
  metadata?: PageMetadata;
83
82
  depth?: number;
84
83
  }
85
- //#endregion
86
- //#region src/crawl.d.ts
87
84
  interface CrawlProgress {
88
85
  sitemap: {
89
86
  status: 'discovering' | 'processing' | 'completed';
@@ -109,5 +106,4 @@ interface CrawlProgress {
109
106
  };
110
107
  }
111
108
  declare function crawlAndGenerate(options: CrawlOptions, onProgress?: (progress: CrawlProgress) => void): Promise<CrawlResult[]>;
112
- //#endregion
113
109
  export { type CrawlHooks, type CrawlOptions, type CrawlResult, type MdreamCrawlConfig, type PageData, crawlAndGenerate, defineConfig };
package/dist/index.mjs CHANGED
@@ -1,7 +1,5 @@
1
1
  import { t as crawlAndGenerate } from "./_chunks/crawl.mjs";
2
- //#region src/types.ts
3
2
  function defineConfig(config) {
4
3
  return config;
5
4
  }
6
- //#endregion
7
5
  export { crawlAndGenerate, defineConfig };
package/package.json CHANGED
@@ -1,7 +1,7 @@
1
1
  {
2
2
  "name": "@mdream/crawl",
3
3
  "type": "module",
4
- "version": "1.0.2",
4
+ "version": "1.0.4",
5
5
  "description": "Mdream Crawl generates comprehensive llms.txt artifacts from a single URL, using mdream to convert HTML to Markdown.",
6
6
  "author": {
7
7
  "name": "Harlan Wilton",
@@ -54,20 +54,20 @@
54
54
  }
55
55
  },
56
56
  "dependencies": {
57
- "@clack/prompts": "^1.1.0",
58
- "c12": "^3.0.4",
59
- "hookable": "^5.5.3",
57
+ "@clack/prompts": "^1.2.0",
58
+ "c12": "^3.3.4",
59
+ "hookable": "^6.1.0",
60
60
  "nypm": "^0.6.5",
61
61
  "ofetch": "^1.5.1",
62
62
  "pathe": "^2.0.3",
63
- "picomatch": "^4.0.3",
64
- "tldts": "^7.0.26",
63
+ "picomatch": "^4.0.4",
64
+ "tldts": "^7.0.28",
65
65
  "ufo": "^1.6.3",
66
- "@mdream/js": "1.0.2",
67
- "mdream": "1.0.2"
66
+ "@mdream/js": "1.0.4",
67
+ "mdream": "1.0.4"
68
68
  },
69
69
  "devDependencies": {
70
- "@types/picomatch": "^4.0.2"
70
+ "@types/picomatch": "^4.0.3"
71
71
  },
72
72
  "scripts": {
73
73
  "build": "obuild",