@mdream/crawl 1.0.0-beta.10 → 1.0.0-beta.11

package/README.md

@@ -7,7 +7,7 @@ Multi-page website crawler that generates comprehensive llms.txt files by follow
 ## Installation
 
 ```bash
-npm install @mdream/crawl
+npm install @mdream/crawl@beta
 ```
 
 ## Usage
@@ -15,7 +15,7 @@ npm install @mdream/crawl
 Simply run the command to start the interactive multi-page website crawler:
 
 ```bash
-npx @mdream/crawl
+npx @mdream/crawl@beta
 ```
 
 The crawler will automatically discover and follow internal links to crawl entire websites. The interactive interface provides:
@@ -46,6 +46,16 @@ const results = await crawlAndGenerate({
 })
 ```
 
+### Playwright Driver
+
+The default HTTP driver works for most sites. For JavaScript-heavy sites that require a browser, install the optional dependencies:
+
+```bash
+npm install crawlee playwright
+```
+
+Then use `--driver playwright` or `driver: 'playwright'` in the API.
+
 > **Note**: llms.txt artifact generation is handled by [`@mdream/js/llms-txt`](../js). The crawl package uses it internally when `generateLlmsTxt: true`.
 
 ## Output
@@ -70,7 +80,7 @@ The crawler generates comprehensive output from entire websites:
 
 - ✅ **Multi-Page Website Crawling**: Designed specifically for crawling entire websites by following internal links
 - ✅ **Purely Interactive**: No complex command-line options to remember
-- ✅ **Dual Crawler Support**: Fast HTTP crawler (default) + Playwright for JavaScript-heavy sites
+- ✅ **Dual Crawler Support**: Fast HTTP crawler (default) + Playwright for JavaScript-heavy sites (requires `crawlee` and `playwright`)
 - ✅ **Smart Link Discovery**: Uses mdream's extraction plugin to find and follow internal links
 - ✅ **Rich Metadata Extraction**: Extracts titles, descriptions, keywords, and author info from all pages
 - ✅ **Comprehensive llms.txt Generation**: Creates complete site documentation files

package/dist/_chunks/crawl.mjs

@@ -1,9 +1,9 @@
-import { existsSync, mkdirSync } from "node:fs";
-import { writeFile } from "node:fs/promises";
+import { mkdirSync } from "node:fs";
+import { mkdir, writeFile } from "node:fs/promises";
 import * as p from "@clack/prompts";
 import { generateLlmsTxtArtifacts } from "@mdream/js/llms-txt";
-import { HttpCrawler, PlaywrightCrawler, log, purgeDefaultStorages } from "crawlee";
 import { htmlToMarkdown } from "mdream";
+import { ofetch } from "ofetch";
 import { dirname, join, normalize, resolve } from "pathe";
 import { withHttps } from "ufo";
 import picomatch from "picomatch";
@@ -109,21 +109,71 @@ function validateGlobPattern(pattern) {
 	}
 }
 //#endregion
-//#region src/metadata-extractor.ts
-function extractMetadata(html, url) {
-	const links = [];
+//#region src/crawl.ts
+const SITEMAP_INDEX_LOC_RE = /<sitemap[^>]*>.*?<loc>(.*?)<\/loc>.*?<\/sitemap>/gs;
+const SITEMAP_URL_LOC_RE = /<url[^>]*>.*?<loc>(.*?)<\/loc>.*?<\/url>/gs;
+const ROBOTS_SITEMAP_RE = /Sitemap:\s*(.*)/gi;
+const ROBOTS_SITEMAP_PREFIX_RE = /Sitemap:\s*/i;
+const ROBOTS_CRAWL_DELAY_RE = /Crawl-delay:\s*(\d+(?:\.\d+)?)/i;
+const URL_TRAILING_SLASH_RE = /\/$/;
+const URL_PATH_UNSAFE_CHARS_RE = /[^\w\-]/g;
+const FRONTMATTER_BLOCK_RE = /^---[^\n]*\n[\s\S]*?\n---[^\n]*\n?/;
+const FETCH_HEADERS = {
+	"User-Agent": "mdream-crawler/1.0",
+	"Accept": "text/html,application/xhtml+xml,text/markdown"
+};
+const DEFAULT_CONCURRENCY = 20;
+function extractCdataUrl(url) {
+	if (url.startsWith("<![CDATA[") && url.endsWith("]]>")) return url.slice(9, -3);
+	return url;
+}
+async function loadSitemap(sitemapUrl) {
+	const xmlContent = await ofetch(sitemapUrl, {
+		headers: FETCH_HEADERS,
+		timeout: 1e4,
+		responseType: "text",
+		retry: 0
+	});
+	if (!isValidSitemapXml(xmlContent)) throw new Error("Response is not a valid sitemap XML");
+	if (xmlContent.includes("<sitemapindex")) {
+		SITEMAP_INDEX_LOC_RE.lastIndex = 0;
+		const childSitemaps = [];
+		let match;
+		while (true) {
+			match = SITEMAP_INDEX_LOC_RE.exec(xmlContent);
+			if (match === null) break;
+			childSitemaps.push(extractCdataUrl(match[1]));
+		}
+		const childResults = await Promise.allSettled(childSitemaps.map((url) => loadSitemap(url)));
+		const allUrls = [];
+		for (const result of childResults) if (result.status === "fulfilled") allUrls.push(...result.value);
+		return allUrls;
+	}
+	const urls = [];
+	SITEMAP_URL_LOC_RE.lastIndex = 0;
+	let match;
+	while (true) {
+		match = SITEMAP_URL_LOC_RE.exec(xmlContent);
+		if (match === null) break;
+		urls.push(extractCdataUrl(match[1]));
+	}
+	return urls;
+}
+function extractMetadataInline(parsedUrl) {
+	const links = /* @__PURE__ */ new Set();
 	let title = "";
 	let description = "";
 	let keywords = "";
 	let author = "";
-	htmlToMarkdown(html, {
-		origin: new URL(url).origin,
+	const url = parsedUrl.href;
+	const originPrefix = `${parsedUrl.origin}/`;
+	return {
 		extraction: {
 			"a[href]": (el) => {
 				const href = el.attributes.href;
 				if (href) try {
 					const absoluteUrl = new URL(href, url).href;
-					if (!links.includes(absoluteUrl)) links.push(absoluteUrl);
+					if (absoluteUrl.startsWith(originPrefix) || absoluteUrl === parsedUrl.origin) links.add(absoluteUrl);
 				} catch {}
 			},
 			"title": (el) => {
@@ -144,88 +194,31 @@ function extractMetadata(html, url) {
 			"meta[property=\"og:title\"]": (el) => {
 				if (!title) title = el.attributes.content || "";
 			}
-		}
-	});
-	return {
-		title: title.trim() || new URL(url).pathname,
-		description: description.trim() || void 0,
-		keywords: keywords.trim() || void 0,
-		author: author.trim() || void 0,
-		links: links.filter((link) => {
-			try {
-				const linkUrl = new URL(link);
-				const baseUrl = new URL(url);
-				return linkUrl.hostname === baseUrl.hostname;
-			} catch {
-				return false;
-			}
+		},
+		getMetadata: () => ({
+			title: title.trim() || parsedUrl.pathname,
+			description: description.trim() || void 0,
+			keywords: keywords.trim() || void 0,
+			author: author.trim() || void 0,
+			links: [...links]
 		})
 	};
 }
-//#endregion
-//#region src/crawl.ts
-const SITEMAP_INDEX_LOC_RE = /<sitemap[^>]*>.*?<loc>(.*?)<\/loc>.*?<\/sitemap>/gs;
-const SITEMAP_URL_LOC_RE = /<url[^>]*>.*?<loc>(.*?)<\/loc>.*?<\/url>/gs;
-const ROBOTS_SITEMAP_RE = /Sitemap:\s*(.*)/gi;
-const ROBOTS_SITEMAP_PREFIX_RE = /Sitemap:\s*/i;
-const URL_TRAILING_SLASH_RE = /\/$/;
-const URL_PATH_UNSAFE_CHARS_RE = /[^\w\-]/g;
-const FRONTMATTER_BLOCK_RE = /^---[^\n]*\n[\s\S]*?\n---[^\n]*\n?/;
-async function loadSitemapWithoutRetries(sitemapUrl) {
-	const controller = new AbortController();
-	const timeoutId = setTimeout(() => controller.abort(), 1e4);
-	try {
-		const response = await fetch(sitemapUrl, {
-			signal: controller.signal,
-			headers: { "User-Agent": "mdream-crawler/1.0" }
-		});
-		clearTimeout(timeoutId);
-		if (!response.ok) throw new Error(`Sitemap not found: ${response.status}`);
-		const xmlContent = await response.text();
-		if (!isValidSitemapXml(xmlContent)) throw new Error("Response is not a valid sitemap XML");
-		if (xmlContent.includes("<sitemapindex")) {
-			SITEMAP_INDEX_LOC_RE.lastIndex = 0;
-			const childSitemaps = [];
-			let match;
-			while (true) {
-				match = SITEMAP_INDEX_LOC_RE.exec(xmlContent);
-				if (match === null) break;
-				let url = match[1];
-				if (url.startsWith("<![CDATA[") && url.endsWith("]]>")) url = url.slice(9, -3);
-				childSitemaps.push(url);
-			}
-			const allUrls = [];
-			for (const childSitemapUrl of childSitemaps) try {
-				const childUrls = await loadSitemapWithoutRetries(childSitemapUrl);
-				allUrls.push(...childUrls);
-			} catch (error) {
-				console.warn(`Failed to load child sitemap ${childSitemapUrl}:`, error instanceof Error ? error.message : "Unknown error");
-			}
-			return allUrls;
-		} else {
-			const urls = [];
-			SITEMAP_URL_LOC_RE.lastIndex = 0;
-			let match;
-			while (true) {
-				match = SITEMAP_URL_LOC_RE.exec(xmlContent);
-				if (match === null) break;
-				let url = match[1];
-				if (url.startsWith("<![CDATA[") && url.endsWith("]]>")) url = url.slice(9, -3);
-				urls.push(url);
-			}
-			return urls;
-		}
-	} catch (error) {
-		clearTimeout(timeoutId);
-		if (error instanceof Error && error.name === "AbortError") throw new Error("Sitemap request timed out after 10 seconds");
-		throw error;
-	}
+function filterSitemapUrls(sitemapUrls, hasGlobPatterns, exclude, allPatterns) {
+	if (hasGlobPatterns) return sitemapUrls.filter((url) => !isUrlExcluded(url, exclude) && allPatterns.some((pattern) => matchesGlobPattern(url, pattern)));
+	return sitemapUrls.filter((url) => !isUrlExcluded(url, exclude));
+}
+async function runConcurrent(items, concurrency, fn) {
+	let idx = 0;
+	const workers = Array.from({ length: Math.min(concurrency, items.length) }, async () => {
+		while (idx < items.length) await fn(items[idx++]);
+	});
+	await Promise.all(workers);
 }
 async function crawlAndGenerate(options, onProgress) {
-	const { urls, outputDir: rawOutputDir, maxRequestsPerCrawl = Number.MAX_SAFE_INTEGER, generateLlmsTxt = true, generateLlmsFullTxt = false, generateIndividualMd = true, origin, driver = "http", useChrome, followLinks = false, maxDepth = 1, globPatterns = [], crawlDelay, exclude = [], siteNameOverride, descriptionOverride, verbose = false, skipSitemap = false, onPage } = options;
+	const { urls, outputDir: rawOutputDir, maxRequestsPerCrawl = Number.MAX_SAFE_INTEGER, generateLlmsTxt = true, generateLlmsFullTxt = false, generateIndividualMd = true, origin, driver = "http", useChrome, followLinks = false, maxDepth = 1, globPatterns = [], crawlDelay: userCrawlDelay, exclude = [], siteNameOverride, descriptionOverride, verbose = false, skipSitemap = false, onPage } = options;
 	const outputDir = resolve(normalize(rawOutputDir));
-	if (verbose) log.setLevel(log.LEVELS.INFO);
-	else log.setLevel(log.LEVELS.OFF);
+	let crawlDelay = userCrawlDelay;
 	let patterns;
 	try {
 		patterns = globPatterns.length > 0 ? globPatterns : urls.map(parseUrlPattern);
@@ -233,6 +226,7 @@ async function crawlAndGenerate(options, onProgress) {
 		throw new Error(`Invalid URL pattern: ${error instanceof Error ? error.message : "Unknown error"}`);
 	}
 	let startingUrls = patterns.map(getStartingUrl);
+	const hasGlobPatterns = patterns.some((p) => p.isGlob);
 	const progress = {
 		sitemap: {
 			status: "discovering",
@@ -242,7 +236,14 @@ async function crawlAndGenerate(options, onProgress) {
 		crawling: {
 			status: "starting",
 			total: 0,
-			processed: 0
+			processed: 0,
+			failed: 0,
+			latency: {
+				total: 0,
+				min: Infinity,
+				max: 0,
+				count: 0
+			}
 		},
 		generation: { status: "idle" }
 	};
@@ -251,51 +252,46 @@ async function crawlAndGenerate(options, onProgress) {
 		const baseUrl = new URL(startingUrls[0]).origin;
 		const homePageUrl = baseUrl;
 		onProgress?.(progress);
-		const robotsUrl = new URL("/robots.txt", baseUrl).toString();
-		const robotsController = new AbortController();
-		const robotsTimeoutId = setTimeout(() => robotsController.abort(), 1e4);
-		let robotsResponse;
+		let robotsContent = null;
 		try {
-			robotsResponse = await fetch(robotsUrl, {
-				signal: robotsController.signal,
-				headers: { "User-Agent": "mdream-crawler/1.0" }
+			robotsContent = await ofetch(`${baseUrl}/robots.txt`, {
+				headers: FETCH_HEADERS,
+				timeout: 1e4,
+				responseType: "text",
+				retry: 0
 			});
-			clearTimeout(robotsTimeoutId);
-		} catch {
-			clearTimeout(robotsTimeoutId);
-			robotsResponse = null;
+		} catch {}
+		if (robotsContent && !crawlDelay) {
+			const crawlDelayMatch = robotsContent.match(ROBOTS_CRAWL_DELAY_RE);
+			if (crawlDelayMatch) {
+				crawlDelay = Number.parseFloat(crawlDelayMatch[1]);
+				p.log(`[ROBOTS] Crawl-delay: ${crawlDelay}s`);
+			}
 		}
-		if (robotsResponse?.ok) {
-			const sitemapMatches = (await robotsResponse.text()).match(ROBOTS_SITEMAP_RE);
+		if (robotsContent) {
+			const sitemapMatches = robotsContent.match(ROBOTS_SITEMAP_RE);
 			if (sitemapMatches && sitemapMatches.length > 0) {
 				progress.sitemap.found = sitemapMatches.length;
 				progress.sitemap.status = "processing";
 				onProgress?.(progress);
 				const robotsSitemaps = sitemapMatches.map((match) => match.replace(ROBOTS_SITEMAP_PREFIX_RE, "").trim());
 				for (const sitemapUrl of robotsSitemaps) try {
-					const robotsUrls = await loadSitemapWithoutRetries(sitemapUrl);
+					const robotsUrls = await loadSitemap(sitemapUrl);
 					sitemapAttempts.push({
 						url: sitemapUrl,
 						success: true
 					});
-					if (patterns.some((p) => p.isGlob)) {
-						const filteredUrls = robotsUrls.filter((url) => {
-							return !isUrlExcluded(url, exclude) && patterns.some((pattern) => matchesGlobPattern(url, pattern));
-						});
+					const filteredUrls = filterSitemapUrls(robotsUrls, hasGlobPatterns, exclude, patterns);
+					if (hasGlobPatterns) {
+						startingUrls = filteredUrls;
+						progress.sitemap.processed = filteredUrls.length;
+						onProgress?.(progress);
+						break;
+					} else if (filteredUrls.length > 0) {
 						startingUrls = filteredUrls;
 						progress.sitemap.processed = filteredUrls.length;
 						onProgress?.(progress);
 						break;
-					} else {
-						const filteredUrls = robotsUrls.filter((url) => {
-							return !isUrlExcluded(url, exclude);
-						});
-						if (filteredUrls.length > 0) {
-							startingUrls = filteredUrls;
-							progress.sitemap.processed = filteredUrls.length;
-							onProgress?.(progress);
-							break;
-						}
 					}
 				} catch (error) {
 					sitemapAttempts.push({
@@ -309,31 +305,24 @@ async function crawlAndGenerate(options, onProgress) {
 		let mainSitemapProcessed = false;
 		const mainSitemapUrl = `${baseUrl}/sitemap.xml`;
 		try {
-			const sitemapUrls = await loadSitemapWithoutRetries(mainSitemapUrl);
+			const sitemapUrls = await loadSitemap(mainSitemapUrl);
 			sitemapAttempts.push({
 				url: mainSitemapUrl,
 				success: true
 			});
-			if (patterns.some((p) => p.isGlob)) {
-				const filteredUrls = sitemapUrls.filter((url) => {
-					return !isUrlExcluded(url, exclude) && patterns.some((pattern) => matchesGlobPattern(url, pattern));
-				});
+			const filteredUrls = filterSitemapUrls(sitemapUrls, hasGlobPatterns, exclude, patterns);
+			if (hasGlobPatterns) {
+				startingUrls = filteredUrls;
+				progress.sitemap.found = sitemapUrls.length;
+				progress.sitemap.processed = filteredUrls.length;
+				onProgress?.(progress);
+				mainSitemapProcessed = true;
+			} else if (filteredUrls.length > 0) {
 				startingUrls = filteredUrls;
 				progress.sitemap.found = sitemapUrls.length;
 				progress.sitemap.processed = filteredUrls.length;
 				onProgress?.(progress);
 				mainSitemapProcessed = true;
-			} else {
-				const filteredUrls = sitemapUrls.filter((url) => {
-					return !isUrlExcluded(url, exclude);
-				});
-				if (filteredUrls.length > 0) {
-					startingUrls = filteredUrls;
-					progress.sitemap.found = sitemapUrls.length;
-					progress.sitemap.processed = filteredUrls.length;
-					onProgress?.(progress);
-					mainSitemapProcessed = true;
-				}
 			}
 		} catch (error) {
 			sitemapAttempts.push({
@@ -348,31 +337,24 @@ async function crawlAndGenerate(options, onProgress) {
 					`${baseUrl}/sitemap-index.xml`
 				];
 				for (const sitemapUrl of commonSitemaps) try {
-					const altUrls = await loadSitemapWithoutRetries(sitemapUrl);
+					const altUrls = await loadSitemap(sitemapUrl);
 					sitemapAttempts.push({
 						url: sitemapUrl,
 						success: true
 					});
-					if (patterns.some((p) => p.isGlob)) {
-						const filteredUrls = altUrls.filter((url) => {
-							return !isUrlExcluded(url, exclude) && patterns.some((pattern) => matchesGlobPattern(url, pattern));
-						});
+					const filteredUrls = filterSitemapUrls(altUrls, hasGlobPatterns, exclude, patterns);
+					if (hasGlobPatterns) {
+						startingUrls = filteredUrls;
+						progress.sitemap.found = altUrls.length;
+						progress.sitemap.processed = filteredUrls.length;
+						onProgress?.(progress);
+						break;
+					} else if (filteredUrls.length > 0) {
 						startingUrls = filteredUrls;
 						progress.sitemap.found = altUrls.length;
 						progress.sitemap.processed = filteredUrls.length;
 						onProgress?.(progress);
 						break;
-					} else {
-						const filteredUrls = altUrls.filter((url) => {
-							return !isUrlExcluded(url, exclude);
-						});
-						if (filteredUrls.length > 0) {
-							startingUrls = filteredUrls;
-							progress.sitemap.found = altUrls.length;
-							progress.sitemap.processed = filteredUrls.length;
-							onProgress?.(progress);
-							break;
-						}
 					}
 				} catch (error) {
 					sitemapAttempts.push({
@@ -405,151 +387,193 @@ async function crawlAndGenerate(options, onProgress) {
 		progress.crawling.total = startingUrls.length;
 		onProgress?.(progress);
 	}
-	if (!existsSync(outputDir)) mkdirSync(outputDir, { recursive: true });
+	mkdirSync(outputDir, { recursive: true });
 	const results = [];
 	const processedUrls = /* @__PURE__ */ new Set();
 	const shouldCrawlUrl = (url) => {
 		if (isUrlExcluded(url, exclude)) return false;
-		if (!patterns.some((p) => p.isGlob)) return true;
+		if (!hasGlobPatterns) return true;
 		return patterns.some((pattern) => matchesGlobPattern(url, pattern));
 	};
-	const createRequestHandler = (crawlerType) => {
-		return async ({ request, body, page, enqueueLinks, response }) => {
-			const startTime = Date.now();
-			progress.crawling.currentUrl = request.loadedUrl;
-			onProgress?.(progress);
-			if (response?.statusCode && (response.statusCode < 200 || response.statusCode >= 300)) return;
-			const homePageUrl = new URL(startingUrls[0]).origin;
-			let html;
-			let title;
-			if (crawlerType === "playwright") {
-				await page.waitForLoadState("networkidle");
-				title = await page.title();
-				html = await page.innerHTML("html");
-			} else {
-				html = typeof body === "string" ? body : body.toString();
-				title = "";
+	const recordLatency = (ms) => {
+		const lat = progress.crawling.latency;
+		lat.total += ms;
+		lat.count++;
+		if (ms < lat.min) lat.min = ms;
+		if (ms > lat.max) lat.max = ms;
+	};
+	const normalizedHomePageUrl = (startingUrls.length > 0 ? new URL(startingUrls[0]).origin : "").replace(URL_TRAILING_SLASH_RE, "");
+	const createdDirs = /* @__PURE__ */ new Set();
+	const sharedOrigin = origin || "";
+	const processPage = async (url, content, initialTitle, depth, isMarkdown = false) => {
+		const parsedUrl = new URL(url);
+		const shouldProcessMarkdown = shouldCrawlUrl(url);
+		const pageOrigin = sharedOrigin || parsedUrl.origin;
+		let md;
+		let metadata;
+		if (isMarkdown) {
+			md = content;
+			metadata = {
+				title: initialTitle || parsedUrl.pathname,
+				links: []
+			};
+		} else {
+			const { extraction, getMetadata } = extractMetadataInline(parsedUrl);
+			md = htmlToMarkdown(content, {
+				origin: pageOrigin,
+				extraction
+			});
+			metadata = getMetadata();
+		}
+		const title = initialTitle || metadata.title;
+		if (onPage && shouldProcessMarkdown) await onPage({
+			url,
+			html: isMarkdown ? "" : content,
+			title,
+			metadata,
+			origin: pageOrigin
+		});
+		let filePath;
+		if (shouldProcessMarkdown && generateIndividualMd) {
+			const safeSegments = (parsedUrl.pathname === "/" ? "/index" : parsedUrl.pathname).replace(URL_TRAILING_SLASH_RE, "").split("/").filter((seg) => seg.length > 0).map((seg) => seg.replace(URL_PATH_UNSAFE_CHARS_RE, "-"));
+			filePath = join(outputDir, normalize(`${safeSegments.length > 0 ? safeSegments.join("/") : "index"}.md`));
+			const fileDir = dirname(filePath);
+			if (fileDir && !createdDirs.has(fileDir)) {
+				await mkdir(fileDir, { recursive: true });
+				createdDirs.add(fileDir);
 			}
-			const metadata = extractMetadata(html, request.loadedUrl);
-			if (!title) title = metadata.title;
-			const shouldProcessMarkdown = shouldCrawlUrl(request.loadedUrl);
-			const pageOrigin = origin || new URL(request.loadedUrl).origin;
-			if (onPage && shouldProcessMarkdown) await onPage({
-				url: request.loadedUrl,
-				html,
+			await writeFile(filePath, md, "utf-8");
+		}
+		const isHomePage = parsedUrl.pathname === "/" && parsedUrl.origin === normalizedHomePageUrl;
+		if (shouldProcessMarkdown || isHomePage) {
+			const result = {
+				url,
 				title,
+				content: md,
+				filePath: shouldProcessMarkdown ? filePath : void 0,
+				timestamp: Date.now(),
+				success: true,
 				metadata,
-				origin: pageOrigin
-			});
-			let md = "";
-			if (shouldProcessMarkdown) md = htmlToMarkdown(html, { origin: pageOrigin });
-			let filePath;
-			if (shouldProcessMarkdown && generateIndividualMd) {
-				const urlObj = new URL(request.loadedUrl);
-				const safeSegments = (urlObj.pathname === "/" ? "/index" : urlObj.pathname).replace(URL_TRAILING_SLASH_RE, "").split("/").filter((seg) => seg.length > 0).map((seg) => seg.replace(URL_PATH_UNSAFE_CHARS_RE, "-"));
-				filePath = join(outputDir, normalize(`${safeSegments.length > 0 ? safeSegments.join("/") : "index"}.md`));
-				const fileDir = dirname(filePath);
-				if (fileDir && !existsSync(fileDir)) mkdirSync(fileDir, { recursive: true });
-				await writeFile(filePath, md, "utf-8");
-			}
-			const isHomePage = request.loadedUrl.replace(URL_TRAILING_SLASH_RE, "") === homePageUrl.replace(URL_TRAILING_SLASH_RE, "");
-			if (shouldProcessMarkdown || isHomePage) {
-				const result = {
-					url: request.loadedUrl,
-					title,
-					content: md,
-					filePath: shouldProcessMarkdown ? filePath : void 0,
-					timestamp: startTime,
-					success: true,
-					metadata,
-					depth: request.userData?.depth || 0
-				};
-				results.push(result);
-				progress.crawling.processed = results.length;
-				onProgress?.(progress);
-			}
-			if (followLinks && (request.userData?.depth || 0) < maxDepth) {
-				const currentDepth = (request.userData?.depth || 0) + 1;
-				const filteredLinks = metadata.links.filter((link) => {
-					return shouldCrawlUrl(link);
-				});
-				if (enqueueLinks) await enqueueLinks({
-					urls: filteredLinks,
-					userData: { depth: currentDepth }
-				});
-				else for (const link of filteredLinks) if (!processedUrls.has(link)) processedUrls.add(link);
-			}
-		};
+				depth
+			};
+			results.push(result);
+			progress.crawling.processed = results.length;
+			onProgress?.(progress);
+		}
+		if (followLinks && depth < maxDepth) {
+			const filteredLinks = metadata.links.filter((link) => shouldCrawlUrl(link));
+			for (const link of filteredLinks) processedUrls.add(link);
+		}
 	};
-	let crawler;
-	const crawlerOptions = {
-		requestHandler: createRequestHandler(driver),
-		errorHandler: async ({ request, response, error }) => {
-			if (verbose) console.error(`[ERROR] URL: ${request.url}, Status: ${response?.statusCode || "N/A"}, Error: ${error?.message || "Unknown"}`);
-			if (response?.statusCode && response?.statusCode >= 400) {
-				request.noRetry = true;
-				const result = {
-					url: request.url,
-					title: "",
-					content: "",
-					timestamp: Date.now(),
-					success: false,
-					error: `HTTP ${response.statusCode}`,
-					metadata: {
-						title: "",
-						description: "",
-						links: []
-					},
-					depth: request.userData?.depth || 0
-				};
-				results.push(result);
-			} else if (error) {
+	const urlsToProcess = startingUrls.slice(0, maxRequestsPerCrawl);
+	progress.crawling.status = "processing";
+	progress.crawling.total = urlsToProcess.length;
+	onProgress?.(progress);
+	if (driver === "playwright") {
+		const { log, PlaywrightCrawler, purgeDefaultStorages } = await import("crawlee");
+		if (verbose) log.setLevel(log.LEVELS.INFO);
+		else log.setLevel(log.LEVELS.OFF);
+		const crawlerOptions = {
+			requestHandler: async ({ request, page }) => {
+				progress.crawling.currentUrl = request.loadedUrl;
+				onProgress?.(progress);
+				const fetchStart = Date.now();
+				await page.waitForLoadState("networkidle");
+				const title = await page.title();
+				const html = await page.innerHTML("html");
+				recordLatency(Date.now() - fetchStart);
+				await processPage(request.loadedUrl, html, title, request.userData?.depth || 0);
+			},
+			errorHandler: async ({ request, response, error }) => {
+				if (verbose) console.error(`[ERROR] URL: ${request.url}, Status: ${response?.statusCode || "N/A"}, Error: ${error?.message || "Unknown"}`);
 				request.noRetry = true;
-				const result = {
+				progress.crawling.failed++;
+				results.push({
 					url: request.url,
 					title: "",
 					content: "",
 					timestamp: Date.now(),
 					success: false,
-					error: error.message || "Unknown error",
+					error: response?.statusCode ? `HTTP ${response.statusCode}` : error?.message || "Unknown error",
 					metadata: {
 						title: "",
 						description: "",
 						links: []
 					},
 					depth: request.userData?.depth || 0
-				};
-				results.push(result);
-			}
-		},
-		maxRequestsPerCrawl,
-		respectRobotsTxtFile: false
-	};
-	if (crawlDelay) crawlerOptions.requestHandlerTimeoutSecs = crawlDelay;
-	if (driver === "playwright") {
-		const playwrightOptions = crawlerOptions;
-		if (useChrome) playwrightOptions.launchContext = {
-			...playwrightOptions.launchContext,
+				});
+			},
+			maxRequestsPerCrawl,
+			respectRobotsTxtFile: false
+		};
+		if (crawlDelay) crawlerOptions.requestHandlerTimeoutSecs = crawlDelay;
+		if (useChrome) crawlerOptions.launchContext = {
+			...crawlerOptions.launchContext,
 			useChrome
 		};
-		crawler = new PlaywrightCrawler(playwrightOptions);
-	} else crawler = new HttpCrawler(crawlerOptions);
-	const initialRequests = startingUrls.map((url) => ({
-		url,
-		userData: { depth: 0 }
-	}));
-	progress.crawling.status = "processing";
-	progress.crawling.total = startingUrls.length;
-	onProgress?.(progress);
-	try {
-		await crawler.run(initialRequests);
-	} catch (error) {
-		if (verbose) {
-			console.error(`[CRAWLER ERROR] ${error instanceof Error ? error.message : "Unknown error"}`);
-			console.error(`[CRAWLER ERROR] Stack trace:`, error instanceof Error ? error.stack : "No stack trace");
+		const crawler = new PlaywrightCrawler(crawlerOptions);
+		const initialRequests = urlsToProcess.map((url) => ({
+			url,
+			userData: { depth: 0 }
+		}));
+		try {
+			await crawler.run(initialRequests);
+		} catch (error) {
+			if (verbose) {
+				console.error(`[CRAWLER ERROR] ${error instanceof Error ? error.message : "Unknown error"}`);
+				console.error(`[CRAWLER ERROR] Stack trace:`, error instanceof Error ? error.stack : "No stack trace");
+			}
+			throw error;
 		}
-		throw error;
-	}
+		await purgeDefaultStorages();
+	} else await runConcurrent(urlsToProcess, DEFAULT_CONCURRENCY, async (url) => {
+		progress.crawling.currentUrl = url;
+		onProgress?.(progress);
+		if (crawlDelay) {
+			const delay = crawlDelay;
+			await new Promise((resolve) => setTimeout(resolve, delay * 1e3));
+		}
+		try {
+			const fetchStart = Date.now();
+			const response = await ofetch.raw(url, {
+				headers: FETCH_HEADERS,
+				responseType: "text",
+				retry: 2,
+				retryDelay: 500,
+				timeout: 1e4,
+				onResponseError({ response }) {
+					if (response.status === 429) {
+						const retryAfter = response.headers.get("retry-after");
+						const delaySec = retryAfter ? Number.parseInt(retryAfter) || 1 : 2;
+						if (!crawlDelay || delaySec > crawlDelay) crawlDelay = delaySec;
+					}
+				}
+			});
+			recordLatency(Date.now() - fetchStart);
+			const body = response._data ?? "";
+			const contentType = response.headers.get("content-type") || "";
+			await processPage(url, body, "", 0, contentType.includes("text/markdown") || contentType.includes("text/x-markdown"));
+		} catch (error) {
+			if (verbose) console.error(`[ERROR] URL: ${url}, Error: ${error instanceof Error ? error.message : "Unknown"}`);
+			progress.crawling.failed++;
+			results.push({
+				url,
+				title: "",
+				content: "",
+				timestamp: Date.now(),
+				success: false,
+				error: error instanceof Error ? error.message : "Unknown error",
+				metadata: {
+					title: "",
+					description: "",
+					links: []
+				},
+				depth: 0
+			});
+			progress.crawling.processed = results.length;
+			onProgress?.(progress);
+		}
+	});
 	progress.crawling.status = "completed";
 	onProgress?.(progress);
 	if (results.some((r) => r.success)) {
@@ -557,10 +581,10 @@ async function crawlAndGenerate(options, onProgress) {
 		onProgress?.(progress);
 		const successfulResults = results.filter((r) => r.success);
 		const firstUrl = new URL(withHttps(urls[0]));
-		const origin = firstUrl.origin;
+		const originUrl = firstUrl.origin;
 		const homePageResult = successfulResults.find((r) => {
 			const resultUrl = new URL(withHttps(r.url));
-			return resultUrl.href === origin || resultUrl.href === `${origin}/`;
+			return resultUrl.href === originUrl || resultUrl.href === `${originUrl}/`;
 		});
 		const siteName = siteNameOverride || homePageResult?.metadata?.title || homePageResult?.title || firstUrl.hostname;
 		const description = descriptionOverride || homePageResult?.metadata?.description || successfulResults[0]?.metadata?.description;
@@ -586,7 +610,7 @@ async function crawlAndGenerate(options, onProgress) {
 				})),
 				siteName,
 				description,
-				origin: origin || firstUrl.origin,
+				origin: originUrl || firstUrl.origin,
 				generateFull: generateLlmsFullTxt,
 				outputDir
 			});
@@ -604,7 +628,6 @@ async function crawlAndGenerate(options, onProgress) {
 		progress.generation.status = "completed";
 		onProgress?.(progress);
 	}
-	await purgeDefaultStorages();
 	return results;
 }
 //#endregion

package/dist/_chunks/playwright-utils.mjs

@@ -0,0 +1,59 @@
+import * as p from "@clack/prompts";
+import { addDependency } from "nypm";
+//#region src/playwright-utils.ts
+async function checkPlaywrightInstallation() {
+	try {
+		await import("playwright");
+		return true;
+	} catch {
+		return false;
+	}
+}
+async function promptPlaywrightInstall() {
+	const shouldInstall = await p.confirm({
+		message: "Playwright is required for the Playwright driver. Install it now?",
+		initialValue: true
+	});
+	if (p.isCancel(shouldInstall) || !shouldInstall) return false;
+	const s = p.spinner();
+	s.start("Installing Playwright globally...");
+	try {
+		await addDependency("playwright", { global: true });
+		s.stop("Playwright installed successfully!");
+		return true;
+	} catch (fallbackError) {
+		s.stop("Failed to install Playwright");
+		p.log.error(`Installation failed: ${fallbackError}`);
+		return false;
+	}
+}
+async function ensurePlaywrightInstalled() {
+	if (await checkPlaywrightInstallation()) return true;
+	p.log.warn("Playwright driver selected but Playwright is not installed.");
+	if (!await promptPlaywrightInstall()) {
+		p.log.error("Cannot proceed with Playwright driver without Playwright installed.");
+		return false;
+	}
+	return true;
+}
+async function isUseChromeSupported() {
+	try {
+		const { PlaywrightCrawler } = await import("crawlee");
+		const crawler = new PlaywrightCrawler({
+			launchContext: { useChrome: true },
+			requestHandler: async () => {},
+			maxRequestsPerCrawl: 1
+		});
+		const page = await crawler.browserPool.newPage();
+		await page.evaluate(() => {
+			return window.navigator.userAgent;
+		});
+		await page.close();
+		await crawler.browserPool.closeAllBrowsers();
+		crawler.stop();
+		return true;
+	} catch {}
+	return false;
+}
+//#endregion
+export { ensurePlaywrightInstalled, isUseChromeSupported };

package/dist/cli.mjs

@@ -1,66 +1,9 @@
 import { n as parseUrlPattern, r as validateGlobPattern, t as crawlAndGenerate } from "./_chunks/crawl.mjs";
 import { accessSync, constants, mkdirSync, readFileSync, unlinkSync, writeFileSync } from "node:fs";
 import * as p from "@clack/prompts";
-import { PlaywrightCrawler } from "crawlee";
 import { dirname, join, resolve } from "pathe";
 import { withHttps } from "ufo";
 import { fileURLToPath } from "node:url";
-import { addDependency } from "nypm";
-//#region src/playwright-utils.ts
-async function checkPlaywrightInstallation() {
-	try {
-		await import("playwright");
-		return true;
-	} catch {
-		return false;
-	}
-}
-async function promptPlaywrightInstall() {
-	const shouldInstall = await p.confirm({
-		message: "Playwright is required for the Playwright driver. Install it now?",
-		initialValue: true
-	});
-	if (p.isCancel(shouldInstall) || !shouldInstall) return false;
-	const s = p.spinner();
-	s.start("Installing Playwright globally...");
-	try {
-		await addDependency("playwright", { global: true });
-		s.stop("Playwright installed successfully!");
-		return true;
-	} catch (fallbackError) {
-		s.stop("Failed to install Playwright");
-		p.log.error(`Installation failed: ${fallbackError}`);
-		return false;
-	}
-}
-async function ensurePlaywrightInstalled() {
-	if (await checkPlaywrightInstallation()) return true;
-	p.log.warn("Playwright driver selected but Playwright is not installed.");
-	if (!await promptPlaywrightInstall()) {
-		p.log.error("Cannot proceed with Playwright driver without Playwright installed.");
-		return false;
-	}
-	return true;
-}
-async function isUseChromeSupported() {
-	try {
-		const crawler = new PlaywrightCrawler({
-			launchContext: { useChrome: true },
-			requestHandler: async () => {},
-			maxRequestsPerCrawl: 1
-		});
-		const page = await crawler.browserPool.newPage();
-		await page.evaluate(() => {
-			return window.navigator.userAgent;
-		});
-		await page.close();
-		await crawler.browserPool.closeAllBrowsers();
-		crawler.stop();
-		return true;
-	} catch {}
-	return false;
-}
-//#endregion
 //#region src/cli.ts
 const packageJsonPath = join(dirname(fileURLToPath(import.meta.url)), "..", "package.json");
 const version = JSON.parse(readFileSync(packageJsonPath, "utf-8")).version;
@@ -225,11 +168,17 @@ async function interactiveCrawl() {
 		skipSitemap: advancedOptions.skipSitemap
 	};
 }
-async function showCrawlResults(successful, failed, outputDir, generatedFiles, durationSeconds) {
+async function showCrawlResults(successful, failed, outputDir, generatedFiles, durationSeconds, latency) {
 	const messages = [];
 	const durationStr = `${durationSeconds.toFixed(1)}s`;
-	const stats = failed > 0 ? `${successful} pages, ${failed} failed` : `${successful} pages`;
-	messages.push(`📄 ${stats} • ⏱️  ${durationStr}`);
+	messages.push(`📄 ${successful} pages \u00B7 ⏱️  ${durationStr}`);
+	if (failed > 0) messages.push(`⚠️  ${failed} failed`);
+	if (latency && latency.count > 0) {
+		const avg = Math.round(latency.total / latency.count);
+		const min = latency.min === Infinity ? 0 : Math.round(latency.min);
+		const max = Math.round(latency.max);
+		messages.push(`🏓 avg ${avg}ms \u00B7 min ${min}ms \u00B7 max ${max}ms`);
+	}
 	messages.push(`📦 ${generatedFiles.join(", ")}`);
 	messages.push(`📁 ${outputDir}`);
 	p.note(messages.join("\n"), "✅ Complete");
@@ -431,36 +380,46 @@ async function main() {
 		if (permCheck.error?.includes("Permission denied")) p.log.info("Tip: Try running with elevated privileges (e.g., sudo) or change the output directory permissions.");
 		process.exit(1);
 	}
-	if (options.driver === "playwright") if (await isUseChromeSupported()) {
-		options.useChrome = true;
-		p.log.info("System Chrome detected and enabled.");
-	} else {
-		if (!await ensurePlaywrightInstalled()) {
-			p.log.error("Cannot proceed without Playwright. Please install it manually or use the HTTP driver instead.");
+	if (options.driver === "playwright") {
+		try {
+			await import("crawlee");
+		} catch {
+			p.log.error("The Playwright driver requires crawlee. Install it with: npm install crawlee");
 			process.exit(1);
 		}
-		p.log.info("Using global playwright instance.");
+		const { ensurePlaywrightInstalled, isUseChromeSupported } = await import("./_chunks/playwright-utils.mjs");
+		if (await isUseChromeSupported()) {
+			options.useChrome = true;
+			p.log.info("System Chrome detected and enabled.");
+		} else {
+			if (!await ensurePlaywrightInstalled()) {
+				p.log.error("Cannot proceed without Playwright. Please install it manually or use the HTTP driver instead.");
+				process.exit(1);
+			}
+			p.log.info("Using global playwright instance.");
+		}
 	}
 	const s = p.spinner();
-	s.start("Starting crawl...");
+	s.start("Discovering sitemaps");
 	const startTime = Date.now();
+	let crawlStartTime = 0;
+	let lastProgress;
 	const results = await crawlAndGenerate(options, (progress) => {
-		if (progress.sitemap.status === "discovering") s.message("Discovering sitemaps...");
+		lastProgress = progress;
+		if (progress.sitemap.status === "discovering") s.message("Discovering sitemaps");
 		else if (progress.sitemap.status === "processing") s.message(`Processing sitemap... Found ${progress.sitemap.found} URLs`);
 		else if (progress.crawling.status === "processing") {
-			const processedCount = progress.crawling.processed;
-			const totalCount = progress.crawling.total;
-			const currentUrl = progress.crawling.currentUrl;
-			if (currentUrl) {
-				const shortUrl = currentUrl.length > 60 ? `${currentUrl.substring(0, 57)}...` : currentUrl;
-				if (processedCount > totalCount) s.message(`Crawling ${processedCount}: ${shortUrl}`);
-				else s.message(`Crawling ${processedCount}/${totalCount}: ${shortUrl}`);
-			} else if (processedCount > totalCount) s.message(`Crawling... ${processedCount} pages`);
-			else s.message(`Crawling... ${processedCount}/${totalCount} pages`);
-		} else if (progress.generation.status === "generating") {
-			const current = progress.generation.current || "Generating files";
-			s.message(current);
-		}
+			if (!crawlStartTime) crawlStartTime = Date.now();
+			const processed = progress.crawling.processed;
+			const total = progress.crawling.total;
+			const failed = progress.crawling.failed;
+			const elapsed = (Date.now() - crawlStartTime) / 1e3;
+			const rate = elapsed > .1 ? Math.round(processed / elapsed) : 0;
+			let msg = processed > total ? `Crawling ${processed} pages` : `Crawling ${processed}/${total}`;
+			if (rate > 0) msg += ` \u00B7 ${rate}/s`;
+			if (failed > 0) msg += ` \u00B7 ${failed} failed`;
+			s.message(msg);
+		} else if (progress.generation.status === "generating") s.message(progress.generation.current || "Generating files");
 	});
 	s.stop();
 	const durationSeconds = (Date.now() - startTime) / 1e3;
@@ -484,7 +443,7 @@ async function main() {
 		if (options.generateLlmsFullTxt) generatedFiles.push("llms-full.txt");
 		if (options.generateIndividualMd) generatedFiles.push(`${successful} MD files`);
 	}
-	await showCrawlResults(successful, failed, options.outputDir, generatedFiles, durationSeconds);
+	await showCrawlResults(successful, failed, options.outputDir, generatedFiles, durationSeconds, lastProgress?.crawling.latency);
 	process.exit(0);
 }
 main().catch((error) => {

package/dist/index.d.mts

@@ -63,7 +63,14 @@ interface CrawlProgress {
     status: 'starting' | 'processing' | 'completed';
     total: number;
     processed: number;
-    currentUrl?: string;
+    failed: number;
+    currentUrl?: string; /** Page fetch latency stats in ms */
+    latency: {
+      total: number;
+      min: number;
+      max: number;
+      count: number;
+    };
   };
   generation: {
     status: 'idle' | 'generating' | 'completed';

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@mdream/crawl",
   "type": "module",
-  "version": "1.0.0-beta.10",
+  "version": "1.0.0-beta.11",
   "description": "Mdream Crawl generates comprehensive llms.txt artifacts from a single URL, using mdream to convert HTML to Markdown.",
   "author": {
     "name": "Harlan Wilton",
@@ -42,22 +42,26 @@
     "dist"
   ],
   "peerDependencies": {
+    "crawlee": "^3.16.0",
     "playwright": "^1.53.2"
   },
   "peerDependenciesMeta": {
+    "crawlee": {
+      "optional": true
+    },
     "playwright": {
       "optional": true
     }
   },
   "dependencies": {
     "@clack/prompts": "^1.1.0",
-    "crawlee": "^3.16.0",
     "nypm": "^0.6.5",
+    "ofetch": "^1.5.1",
     "pathe": "^2.0.3",
     "picomatch": "^4.0.3",
     "ufo": "^1.6.3",
-    "mdream": "1.0.0-beta.10",
-    "@mdream/js": "1.0.0-beta.10"
+    "@mdream/js": "1.0.0-beta.11",
+    "mdream": "1.0.0-beta.11"
   },
   "devDependencies": {
     "@types/picomatch": "^4.0.2"