@mdream/crawl 0.7.0 → 0.7.2

package/dist/_chunks/{crawl-NJU1Dyc-.mjs → crawl-DEysrw0h.mjs}

@@ -1,10 +1,10 @@
 import { existsSync, mkdirSync } from "node:fs";
 import { writeFile } from "node:fs/promises";
-import { HttpCrawler, Sitemap, purgeDefaultStorages } from "crawlee";
+import { HttpCrawler, PlaywrightCrawler, Sitemap, purgeDefaultStorages } from "crawlee";
 import { generateLlmsTxtArtifacts, htmlToMarkdown } from "mdream";
 import { withMinimalPreset } from "mdream/preset/minimal";
 import { dirname, join, normalize, resolve } from "pathe";
-import { minimatch } from "minimatch";
+import picomatch from "picomatch";
 import { extractionPlugin } from "mdream/plugins";
 
 //#region ../../node_modules/.pnpm/ufo@1.6.1/node_modules/ufo/dist/index.mjs
@@ -60,7 +60,12 @@ function matchesGlobPattern(url, parsedPattern) {
 		const urlPath = urlObj.pathname + urlObj.search + urlObj.hash;
 		const urlBase = `${urlObj.protocol}//${urlObj.host}`;
 		if (urlBase !== parsedPattern.baseUrl) return false;
-		return minimatch(urlPath, parsedPattern.pattern);
+		let pattern = parsedPattern.pattern;
+		if (pattern.endsWith("*") && !pattern.endsWith("**") && !pattern.endsWith("/*")) {
+			const base = pattern.slice(0, -1);
+			pattern = `{${base},${base}/**}`;
+		}
+		return picomatch(pattern)(urlPath);
 	} catch {
 		return false;
 	}
@@ -95,9 +100,9 @@ function isUrlExcluded(url, excludePatterns) {
 			}
 			if (pattern.startsWith("/")) {
 				const adjustedPattern = pattern.endsWith("/*") ? pattern.replace("/*", "/**") : pattern;
-				return minimatch(urlPath, adjustedPattern);
+				return picomatch(adjustedPattern)(urlPath);
 			}
-			return minimatch(urlPath, pattern) || minimatch(urlPath.substring(1), pattern);
+			return picomatch(pattern)(urlPath) || picomatch(pattern)(urlPath.substring(1));
 		});
 	} catch {
 		return false;
@@ -237,6 +242,7 @@ async function crawlAndGenerate(options, onProgress) {
 				}
 			}
 		}
+		let mainSitemapProcessed = false;
 		try {
 			const { urls: sitemapUrls } = await Sitemap.load(`${baseUrl}/sitemap.xml`);
 			const hasGlobPatterns = patterns.some((p) => p.isGlob);
@@ -248,6 +254,7 @@ async function crawlAndGenerate(options, onProgress) {
 				progress.sitemap.found = sitemapUrls.length;
 				progress.sitemap.processed = filteredUrls.length;
 				onProgress?.(progress);
+				mainSitemapProcessed = true;
 			} else {
 				const filteredUrls = sitemapUrls.filter((url) => {
 					return !isUrlExcluded(url, exclude);
@@ -257,40 +264,43 @@ async function crawlAndGenerate(options, onProgress) {
 					progress.sitemap.found = sitemapUrls.length;
 					progress.sitemap.processed = filteredUrls.length;
 					onProgress?.(progress);
+					mainSitemapProcessed = true;
 				}
 			}
 		} catch {
-			const commonSitemaps = [
-				`${baseUrl}/sitemap_index.xml`,
-				`${baseUrl}/sitemaps.xml`,
-				`${baseUrl}/sitemap-index.xml`
-			];
-			for (const sitemapUrl of commonSitemaps) try {
-				const { urls: altUrls } = await Sitemap.load(sitemapUrl);
-				const hasGlobPatterns = patterns.some((p) => p.isGlob);
-				if (hasGlobPatterns) {
-					const filteredUrls = altUrls.filter((url) => {
-						return !isUrlExcluded(url, exclude) && patterns.some((pattern) => matchesGlobPattern(url, pattern));
-					});
-					startingUrls = filteredUrls;
-					progress.sitemap.found = altUrls.length;
-					progress.sitemap.processed = filteredUrls.length;
-					onProgress?.(progress);
-					break;
-				} else {
-					const filteredUrls = altUrls.filter((url) => {
-						return !isUrlExcluded(url, exclude);
-					});
-					if (filteredUrls.length > 0) {
+			if (!mainSitemapProcessed) {
+				const commonSitemaps = [
+					`${baseUrl}/sitemap_index.xml`,
+					`${baseUrl}/sitemaps.xml`,
+					`${baseUrl}/sitemap-index.xml`
+				];
+				for (const sitemapUrl of commonSitemaps) try {
+					const { urls: altUrls } = await Sitemap.load(sitemapUrl);
+					const hasGlobPatterns = patterns.some((p) => p.isGlob);
+					if (hasGlobPatterns) {
+						const filteredUrls = altUrls.filter((url) => {
+							return !isUrlExcluded(url, exclude) && patterns.some((pattern) => matchesGlobPattern(url, pattern));
+						});
 						startingUrls = filteredUrls;
 						progress.sitemap.found = altUrls.length;
 						progress.sitemap.processed = filteredUrls.length;
 						onProgress?.(progress);
 						break;
+					} else {
+						const filteredUrls = altUrls.filter((url) => {
+							return !isUrlExcluded(url, exclude);
+						});
+						if (filteredUrls.length > 0) {
+							startingUrls = filteredUrls;
+							progress.sitemap.found = altUrls.length;
+							progress.sitemap.processed = filteredUrls.length;
+							onProgress?.(progress);
+							break;
+						}
 					}
+				} catch {
+					continue;
 				}
-			} catch {
-				continue;
 			}
 		}
 		if (!startingUrls.includes(homePageUrl)) startingUrls.unshift(homePageUrl);
@@ -311,8 +321,7 @@ async function crawlAndGenerate(options, onProgress) {
 			const startTime = Date.now();
 			progress.crawling.currentUrl = request.loadedUrl;
 			onProgress?.(progress);
-			const baseUrl = new URL(startingUrls[0]).origin;
-			const homePageUrl = baseUrl;
+			const homePageUrl = new URL(startingUrls[0]).origin;
 			let html;
 			let title;
 			if (crawlerType === "playwright") {
@@ -375,14 +384,15 @@ async function crawlAndGenerate(options, onProgress) {
 	let crawler;
 	const crawlerOptions = {
 		requestHandler: createRequestHandler(driver),
+		errorHandler: async ({ request, response }, error) => {
+			if (response?.statusCode && response?.statusCode >= 400) request.noRetry = true;
+		},
 		maxRequestsPerCrawl,
 		respectRobotsTxtFile: true
 	};
-	if (crawlDelay) crawlerOptions.requestHandlerTimeoutMillis = crawlDelay * 1e3;
-	if (driver === "playwright") {
-		const { PlaywrightCrawler: PlaywrightCrawlerClass } = await import("crawlee");
-		crawler = new PlaywrightCrawlerClass(crawlerOptions);
-	} else crawler = new HttpCrawler(crawlerOptions);
+	if (crawlDelay) crawlerOptions.requestHandlerTimeoutSecs = crawlDelay;
+	if (driver === "playwright") crawler = new PlaywrightCrawler(crawlerOptions);
+	else crawler = new HttpCrawler(crawlerOptions);
 	const initialRequests = startingUrls.map((url) => ({
 		url,
 		userData: { depth: 0 }

package/dist/cli.mjs

@@ -1,4 +1,4 @@
-import { crawlAndGenerate, parseUrlPattern, validateGlobPattern, withHttps } from "./_chunks/crawl-NJU1Dyc-.mjs";
+import { crawlAndGenerate, parseUrlPattern, validateGlobPattern, withHttps } from "./_chunks/crawl-DEysrw0h.mjs";
 import { readFileSync } from "node:fs";
 import { dirname, join, resolve } from "pathe";
 import { fileURLToPath } from "node:url";
@@ -197,15 +197,16 @@ async function interactiveCrawl() {
 		globPatterns
 	};
 }
-async function showCrawlResults(successful, failed, outputDir, generatedFiles) {
+async function showCrawlResults(successful, failed, outputDir, generatedFiles, durationMinutes, remainingSeconds) {
 	const messages = [];
-	if (successful > 0) messages.push(`✅ ${successful} pages processed successfully`);
+	if (successful > 0) {
+		const durationStr = durationMinutes > 0 ? `${durationMinutes}m ${remainingSeconds}s` : `${remainingSeconds}s`;
+		messages.push(`✅ ${successful} pages processed in ${durationStr}`);
+	}
 	if (failed > 0) messages.push(`❌ ${failed} pages failed`);
 	if (generatedFiles.length > 0) messages.push(`📄 Generated: ${generatedFiles.join(", ")}`);
 	messages.push(`📁 Output: ${outputDir}`);
-	p.note(messages.join("\n"), "Crawl Results");
-	if (successful > 0) p.outro("🎉 Crawling completed successfully!");
-	else p.outro("❌ Crawling failed - no pages processed");
+	p.note(messages.join("\n"), "Completed: Results");
 }
 function parseCliArgs() {
 	const args = process.argv.slice(2);
@@ -396,6 +397,7 @@ async function main() {
 	}
 	const s = p.spinner();
 	s.start("Starting crawl...");
+	const startTime = Date.now();
 	const results = await crawlAndGenerate(options, (progress) => {
 		if (progress.sitemap.status === "discovering") s.message("Discovering sitemaps...");
 		else if (progress.sitemap.status === "processing") s.message(`Processing sitemap... Found ${progress.sitemap.found} URLs`);
@@ -414,7 +416,12 @@ async function main() {
 			s.message(current);
 		}
 	});
-	s.stop("Crawl completed!");
+	s.stop();
+	const endTime = Date.now();
+	const durationMs = endTime - startTime;
+	const durationSeconds = Math.floor(durationMs / 1e3);
+	const durationMinutes = Math.floor(durationSeconds / 60);
+	const remainingSeconds = durationSeconds % 60;
 	const successful = results.filter((r) => r.success).length;
 	const failed = results.filter((r) => !r.success).length;
 	const failedResults = results.filter((r) => !r.success);
@@ -435,19 +442,18 @@ async function main() {
 		if (options.generateLlmsFullTxt) generatedFiles.push("llms-full.txt");
 		if (options.generateIndividualMd) generatedFiles.push(`${successful} MD files`);
 	}
-	if (!cliOptions) await showCrawlResults(successful, failed, options.outputDir, generatedFiles);
+	if (!cliOptions) await showCrawlResults(successful, failed, options.outputDir, generatedFiles, durationMinutes, remainingSeconds);
 	else {
 		const messages = [];
-		if (successful > 0) messages.push(`✅ ${successful} pages processed`);
+		if (successful > 0) {
+			const durationStr = durationMinutes > 0 ? `${durationMinutes}m ${remainingSeconds}s` : `${remainingSeconds}s`;
+			messages.push(`✅ ${successful} pages processed in ${durationStr}`);
+		}
 		if (failed > 0) messages.push(`❌ ${failed} pages failed`);
 		if (generatedFiles.length > 0) messages.push(`📄 Generated: ${generatedFiles.join(", ")}`);
 		messages.push(`📁 Output: ${options.outputDir}`);
-		p.note(messages.join("\n"), "Results");
-		if (successful > 0) p.outro("🎉 Crawling completed!");
-		else {
-			p.outro("❌ Crawling failed - no pages processed");
-			process.exit(1);
-		}
+		p.note(messages.join("\n"), "Completed: Results");
+		if (successful === 0) process.exit(1);
 	}
 }
 main().catch((error) => {

package/dist/index.mjs

@@ -1,4 +1,4 @@
-import { crawlAndGenerate } from "./_chunks/crawl-NJU1Dyc-.mjs";
+import { crawlAndGenerate } from "./_chunks/crawl-DEysrw0h.mjs";
 import { writeFile } from "node:fs/promises";
 import { basename, sep } from "pathe";
 

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@mdream/crawl",
   "type": "module",
-  "version": "0.7.0",
+  "version": "0.7.2",
   "description": "Mdream Crawl generates comprehensive llms.txt artifacts from a single URL, using mdream to convert HTML to Markdown.",
   "author": {
     "name": "Harlan Wilton",
@@ -46,11 +46,14 @@
   },
   "dependencies": {
     "@clack/prompts": "^0.11.0",
-    "crawlee": "^3.13.9",
-    "minimatch": "^10.0.3",
+    "crawlee": "^3.13.10",
     "nypm": "^0.6.0",
     "pathe": "^2.0.3",
-    "mdream": "0.7.0"
+    "picomatch": "^4.0.2",
+    "mdream": "0.7.2"
+  },
+  "devDependencies": {
+    "@types/picomatch": "^4.0.0"
   },
   "scripts": {
     "build": "obuild",