@mdream/crawl 1.0.0-beta.9 → 1.0.0

package/dist/_chunks/crawl.mjs

@@ -1,12 +1,14 @@
-import { existsSync, mkdirSync } from "node:fs";
-import { writeFile } from "node:fs/promises";
+import { mkdirSync } from "node:fs";
+import { mkdir, writeFile } from "node:fs/promises";
 import * as p from "@clack/prompts";
 import { generateLlmsTxtArtifacts } from "@mdream/js/llms-txt";
-import { HttpCrawler, PlaywrightCrawler, log, purgeDefaultStorages } from "crawlee";
+import { createHooks } from "hookable";
 import { htmlToMarkdown } from "mdream";
+import { ofetch } from "ofetch";
 import { dirname, join, normalize, resolve } from "pathe";
 import { withHttps } from "ufo";
 import picomatch from "picomatch";
+import { getDomain } from "tldts";
 //#region src/glob-utils.ts
 function stripGlobTail(s) {
 	const idx = s.indexOf("*");
@@ -14,6 +16,14 @@ function stripGlobTail(s) {
 }
 const GLOB_CHAR_RE = /[*?[]/;
 /**
+* Extract the registrable domain from a hostname using the public suffix list.
+* Handles multi-part TLDs (.co.uk, .github.io, etc.) correctly.
+* Returns the hostname unchanged for IPs or when parsing fails.
+*/
+function getRegistrableDomain(hostname) {
+	return getDomain(hostname, { allowPrivateDomains: true }) || hostname;
+}
+/**
 * Parse a URL that may contain glob patterns
 * Example: https://nuxtseo.com/docs/** -> { baseUrl: "https://nuxtseo.com", pattern: "/docs/**", isGlob: true }
 */
@@ -40,12 +50,15 @@ function parseUrlPattern(input) {
 /**
 * Check if a URL matches a glob pattern
 */
-function matchesGlobPattern(url, parsedPattern) {
+function matchesGlobPattern(url, parsedPattern, allowSubdomains = false) {
 	if (!parsedPattern.isGlob) return true;
 	try {
 		const urlObj = new URL(url);
 		const urlPath = urlObj.pathname + urlObj.search + urlObj.hash;
-		if (`${urlObj.protocol}//${urlObj.host}` !== parsedPattern.baseUrl) return false;
+		if (allowSubdomains) {
+			const patternUrl = new URL(parsedPattern.baseUrl);
+			if (getRegistrableDomain(urlObj.hostname) !== getRegistrableDomain(patternUrl.hostname)) return false;
+		} else if (`${urlObj.protocol}//${urlObj.host}` !== parsedPattern.baseUrl) return false;
 		let pattern = parsedPattern.pattern;
 		if (pattern.endsWith("*") && !pattern.endsWith("**") && !pattern.endsWith("/*")) {
 			const base = pattern.slice(0, -1);
@@ -73,7 +86,7 @@ function getStartingUrl(parsedPattern) {
 /**
 * Check if a URL should be excluded based on exclude patterns
 */
-function isUrlExcluded(url, excludePatterns) {
+function isUrlExcluded(url, excludePatterns, allowSubdomains = false) {
 	if (!excludePatterns || excludePatterns.length === 0) return false;
 	try {
 		const urlObj = new URL(url);
@@ -81,7 +94,7 @@ function isUrlExcluded(url, excludePatterns) {
 		return excludePatterns.some((pattern) => {
 			if (pattern.includes("://")) {
 				const parsedPattern = parseUrlPattern(pattern);
-				if (parsedPattern.isGlob) return matchesGlobPattern(url, parsedPattern);
+				if (parsedPattern.isGlob) return matchesGlobPattern(url, parsedPattern, allowSubdomains);
 				return url === pattern;
 			}
 			if (pattern.startsWith("/")) return picomatch(pattern.endsWith("/*") ? pattern.replace("/*", "/**") : pattern)(urlPath);
@@ -109,21 +122,75 @@ function validateGlobPattern(pattern) {
 	}
 }
 //#endregion
-//#region src/metadata-extractor.ts
-function extractMetadata(html, url) {
-	const links = [];
+//#region src/crawl.ts
+const SITEMAP_INDEX_LOC_RE = /<sitemap[^>]*>.*?<loc>(.*?)<\/loc>.*?<\/sitemap>/gs;
+const SITEMAP_URL_LOC_RE = /<url[^>]*>.*?<loc>(.*?)<\/loc>.*?<\/url>/gs;
+const ROBOTS_SITEMAP_RE = /Sitemap:\s*(.*)/gi;
+const ROBOTS_SITEMAP_PREFIX_RE = /Sitemap:\s*/i;
+const ROBOTS_CRAWL_DELAY_RE = /Crawl-delay:\s*(\d+(?:\.\d+)?)/i;
+const URL_TRAILING_SLASH_RE = /\/$/;
+const URL_PATH_UNSAFE_CHARS_RE = /[^\w\-]/g;
+const FRONTMATTER_BLOCK_RE = /^---[^\n]*\n[\s\S]*?\n---[^\n]*\n?/;
+const FETCH_HEADERS = {
+	"User-Agent": "mdream-crawler/1.0",
+	"Accept": "text/html,application/xhtml+xml,text/markdown"
+};
+const DEFAULT_CONCURRENCY = 20;
+function extractCdataUrl(url) {
+	if (url.startsWith("<![CDATA[") && url.endsWith("]]>")) return url.slice(9, -3);
+	return url;
+}
+async function loadSitemap(sitemapUrl) {
+	const xmlContent = await ofetch(sitemapUrl, {
+		headers: FETCH_HEADERS,
+		timeout: 1e4,
+		responseType: "text",
+		retry: 0
+	});
+	if (!isValidSitemapXml(xmlContent)) throw new Error("Response is not a valid sitemap XML");
+	if (xmlContent.includes("<sitemapindex")) {
+		SITEMAP_INDEX_LOC_RE.lastIndex = 0;
+		const childSitemaps = [];
+		let match;
+		while (true) {
+			match = SITEMAP_INDEX_LOC_RE.exec(xmlContent);
+			if (match === null) break;
+			childSitemaps.push(extractCdataUrl(match[1]));
+		}
+		const childResults = await Promise.allSettled(childSitemaps.map((url) => loadSitemap(url)));
+		const allUrls = [];
+		for (const result of childResults) if (result.status === "fulfilled") allUrls.push(...result.value);
+		return allUrls;
+	}
+	const urls = [];
+	SITEMAP_URL_LOC_RE.lastIndex = 0;
+	let match;
+	while (true) {
+		match = SITEMAP_URL_LOC_RE.exec(xmlContent);
+		if (match === null) break;
+		urls.push(extractCdataUrl(match[1]));
+	}
+	return urls;
+}
+function extractMetadataInline(parsedUrl, allowedDomains) {
+	const links = /* @__PURE__ */ new Set();
 	let title = "";
 	let description = "";
 	let keywords = "";
 	let author = "";
-	htmlToMarkdown(html, {
-		origin: new URL(url).origin,
+	const url = parsedUrl.href;
+	const originPrefix = `${parsedUrl.origin}/`;
+	return {
 		extraction: {
 			"a[href]": (el) => {
 				const href = el.attributes.href;
 				if (href) try {
-					const absoluteUrl = new URL(href, url).href;
-					if (!links.includes(absoluteUrl)) links.push(absoluteUrl);
+					const resolved = new URL(href, url);
+					const absoluteUrl = resolved.href;
+					if (allowedDomains) {
+						const domain = getRegistrableDomain(resolved.hostname);
+						if (domain && allowedDomains.has(domain)) links.add(absoluteUrl);
+					} else if (absoluteUrl.startsWith(originPrefix) || absoluteUrl === parsedUrl.origin) links.add(absoluteUrl);
 				} catch {}
 			},
 			"title": (el) => {
@@ -144,88 +211,35 @@ function extractMetadata(html, url) {
 			"meta[property=\"og:title\"]": (el) => {
 				if (!title) title = el.attributes.content || "";
 			}
-		}
-	});
-	return {
-		title: title.trim() || new URL(url).pathname,
-		description: description.trim() || void 0,
-		keywords: keywords.trim() || void 0,
-		author: author.trim() || void 0,
-		links: links.filter((link) => {
-			try {
-				const linkUrl = new URL(link);
-				const baseUrl = new URL(url);
-				return linkUrl.hostname === baseUrl.hostname;
-			} catch {
-				return false;
-			}
+		},
+		getMetadata: () => ({
+			title: title.trim() || parsedUrl.pathname,
+			description: description.trim() || void 0,
+			keywords: keywords.trim() || void 0,
+			author: author.trim() || void 0,
+			links: [...links]
 		})
 	};
 }
-//#endregion
-//#region src/crawl.ts
-const SITEMAP_INDEX_LOC_RE = /<sitemap[^>]*>.*?<loc>(.*?)<\/loc>.*?<\/sitemap>/gs;
-const SITEMAP_URL_LOC_RE = /<url[^>]*>.*?<loc>(.*?)<\/loc>.*?<\/url>/gs;
-const ROBOTS_SITEMAP_RE = /Sitemap:\s*(.*)/gi;
-const ROBOTS_SITEMAP_PREFIX_RE = /Sitemap:\s*/i;
-const URL_TRAILING_SLASH_RE = /\/$/;
-const URL_PATH_UNSAFE_CHARS_RE = /[^\w\-]/g;
-const FRONTMATTER_BLOCK_RE = /^---[^\n]*\n[\s\S]*?\n---[^\n]*\n?/;
-async function loadSitemapWithoutRetries(sitemapUrl) {
-	const controller = new AbortController();
-	const timeoutId = setTimeout(() => controller.abort(), 1e4);
-	try {
-		const response = await fetch(sitemapUrl, {
-			signal: controller.signal,
-			headers: { "User-Agent": "mdream-crawler/1.0" }
-		});
-		clearTimeout(timeoutId);
-		if (!response.ok) throw new Error(`Sitemap not found: ${response.status}`);
-		const xmlContent = await response.text();
-		if (!isValidSitemapXml(xmlContent)) throw new Error("Response is not a valid sitemap XML");
-		if (xmlContent.includes("<sitemapindex")) {
-			SITEMAP_INDEX_LOC_RE.lastIndex = 0;
-			const childSitemaps = [];
-			let match;
-			while (true) {
-				match = SITEMAP_INDEX_LOC_RE.exec(xmlContent);
-				if (match === null) break;
-				let url = match[1];
-				if (url.startsWith("<![CDATA[") && url.endsWith("]]>")) url = url.slice(9, -3);
-				childSitemaps.push(url);
-			}
-			const allUrls = [];
-			for (const childSitemapUrl of childSitemaps) try {
-				const childUrls = await loadSitemapWithoutRetries(childSitemapUrl);
-				allUrls.push(...childUrls);
-			} catch (error) {
-				console.warn(`Failed to load child sitemap ${childSitemapUrl}:`, error instanceof Error ? error.message : "Unknown error");
-			}
-			return allUrls;
-		} else {
-			const urls = [];
-			SITEMAP_URL_LOC_RE.lastIndex = 0;
-			let match;
-			while (true) {
-				match = SITEMAP_URL_LOC_RE.exec(xmlContent);
-				if (match === null) break;
-				let url = match[1];
-				if (url.startsWith("<![CDATA[") && url.endsWith("]]>")) url = url.slice(9, -3);
-				urls.push(url);
-			}
-			return urls;
-		}
-	} catch (error) {
-		clearTimeout(timeoutId);
-		if (error instanceof Error && error.name === "AbortError") throw new Error("Sitemap request timed out after 10 seconds");
-		throw error;
-	}
+function filterSitemapUrls(sitemapUrls, hasGlobPatterns, exclude, allPatterns, allowSubdomains = false) {
+	if (hasGlobPatterns) return sitemapUrls.filter((url) => !isUrlExcluded(url, exclude, allowSubdomains) && allPatterns.some((pattern) => matchesGlobPattern(url, pattern, allowSubdomains)));
+	return sitemapUrls.filter((url) => !isUrlExcluded(url, exclude, allowSubdomains));
+}
+async function runConcurrent(items, concurrency, fn) {
+	let idx = 0;
+	const workers = Array.from({ length: Math.min(concurrency, items.length) }, async () => {
+		while (idx < items.length) await fn(items[idx++]);
+	});
+	await Promise.all(workers);
 }
 async function crawlAndGenerate(options, onProgress) {
-	const { urls, outputDir: rawOutputDir, maxRequestsPerCrawl = Number.MAX_SAFE_INTEGER, generateLlmsTxt = true, generateLlmsFullTxt = false, generateIndividualMd = true, origin, driver = "http", useChrome, followLinks = false, maxDepth = 1, globPatterns = [], crawlDelay, exclude = [], siteNameOverride, descriptionOverride, verbose = false, skipSitemap = false, onPage } = options;
+	const { urls, outputDir: rawOutputDir, maxRequestsPerCrawl = Number.MAX_SAFE_INTEGER, generateLlmsTxt = true, generateLlmsFullTxt = false, generateIndividualMd = true, origin, driver = "http", useChrome, followLinks = false, maxDepth = 1, globPatterns = [], crawlDelay: userCrawlDelay, exclude = [], siteNameOverride, descriptionOverride, verbose = false, skipSitemap = false, allowSubdomains = false, hooks: hooksConfig, onPage } = options;
+	const hooks = createHooks();
+	if (hooksConfig) hooks.addHooks(hooksConfig);
+	if (onPage) hooks.hook("crawl:page", onPage);
+	const singlePageMode = maxDepth === 0;
 	const outputDir = resolve(normalize(rawOutputDir));
-	if (verbose) log.setLevel(log.LEVELS.INFO);
-	else log.setLevel(log.LEVELS.OFF);
+	let crawlDelay = userCrawlDelay;
 	let patterns;
 	try {
 		patterns = globPatterns.length > 0 ? globPatterns : urls.map(parseUrlPattern);
@@ -233,6 +247,7 @@ async function crawlAndGenerate(options, onProgress) {
 		throw new Error(`Invalid URL pattern: ${error instanceof Error ? error.message : "Unknown error"}`);
 	}
 	let startingUrls = patterns.map(getStartingUrl);
+	const hasGlobPatterns = patterns.some((p) => p.isGlob);
 	const progress = {
 		sitemap: {
 			status: "discovering",
@@ -242,60 +257,62 @@ async function crawlAndGenerate(options, onProgress) {
 		crawling: {
 			status: "starting",
 			total: 0,
-			processed: 0
+			processed: 0,
+			failed: 0,
+			latency: {
+				total: 0,
+				min: Infinity,
+				max: 0,
+				count: 0
+			}
 		},
 		generation: { status: "idle" }
 	};
 	const sitemapAttempts = [];
-	if (startingUrls.length > 0 && !skipSitemap) {
+	if (startingUrls.length > 0 && !skipSitemap && !singlePageMode) {
 		const baseUrl = new URL(startingUrls[0]).origin;
 		const homePageUrl = baseUrl;
 		onProgress?.(progress);
-		const robotsUrl = new URL("/robots.txt", baseUrl).toString();
-		const robotsController = new AbortController();
-		const robotsTimeoutId = setTimeout(() => robotsController.abort(), 1e4);
-		let robotsResponse;
+		let robotsContent = null;
 		try {
-			robotsResponse = await fetch(robotsUrl, {
-				signal: robotsController.signal,
-				headers: { "User-Agent": "mdream-crawler/1.0" }
+			robotsContent = await ofetch(`${baseUrl}/robots.txt`, {
+				headers: FETCH_HEADERS,
+				timeout: 1e4,
+				responseType: "text",
+				retry: 0
 			});
-			clearTimeout(robotsTimeoutId);
-		} catch {
-			clearTimeout(robotsTimeoutId);
-			robotsResponse = null;
+		} catch {}
+		if (robotsContent && !crawlDelay) {
+			const crawlDelayMatch = robotsContent.match(ROBOTS_CRAWL_DELAY_RE);
+			if (crawlDelayMatch) {
+				crawlDelay = Number.parseFloat(crawlDelayMatch[1]);
+				p.log.info(`[ROBOTS] Crawl-delay: ${crawlDelay}s`);
+			}
 		}
-		if (robotsResponse?.ok) {
-			const sitemapMatches = (await robotsResponse.text()).match(ROBOTS_SITEMAP_RE);
+		if (robotsContent) {
+			const sitemapMatches = robotsContent.match(ROBOTS_SITEMAP_RE);
 			if (sitemapMatches && sitemapMatches.length > 0) {
 				progress.sitemap.found = sitemapMatches.length;
 				progress.sitemap.status = "processing";
 				onProgress?.(progress);
 				const robotsSitemaps = sitemapMatches.map((match) => match.replace(ROBOTS_SITEMAP_PREFIX_RE, "").trim());
 				for (const sitemapUrl of robotsSitemaps) try {
-					const robotsUrls = await loadSitemapWithoutRetries(sitemapUrl);
+					const robotsUrls = await loadSitemap(sitemapUrl);
 					sitemapAttempts.push({
 						url: sitemapUrl,
 						success: true
 					});
-					if (patterns.some((p) => p.isGlob)) {
-						const filteredUrls = robotsUrls.filter((url) => {
-							return !isUrlExcluded(url, exclude) && patterns.some((pattern) => matchesGlobPattern(url, pattern));
-						});
+					const filteredUrls = filterSitemapUrls(robotsUrls, hasGlobPatterns, exclude, patterns, allowSubdomains);
+					if (hasGlobPatterns) {
+						startingUrls = filteredUrls;
+						progress.sitemap.processed = filteredUrls.length;
+						onProgress?.(progress);
+						break;
+					} else if (filteredUrls.length > 0) {
 						startingUrls = filteredUrls;
 						progress.sitemap.processed = filteredUrls.length;
 						onProgress?.(progress);
 						break;
-					} else {
-						const filteredUrls = robotsUrls.filter((url) => {
-							return !isUrlExcluded(url, exclude);
-						});
-						if (filteredUrls.length > 0) {
-							startingUrls = filteredUrls;
-							progress.sitemap.processed = filteredUrls.length;
-							onProgress?.(progress);
-							break;
-						}
 					}
 				} catch (error) {
 					sitemapAttempts.push({
@@ -309,31 +326,24 @@ async function crawlAndGenerate(options, onProgress) {
 		let mainSitemapProcessed = false;
 		const mainSitemapUrl = `${baseUrl}/sitemap.xml`;
 		try {
-			const sitemapUrls = await loadSitemapWithoutRetries(mainSitemapUrl);
+			const sitemapUrls = await loadSitemap(mainSitemapUrl);
 			sitemapAttempts.push({
 				url: mainSitemapUrl,
 				success: true
 			});
-			if (patterns.some((p) => p.isGlob)) {
-				const filteredUrls = sitemapUrls.filter((url) => {
-					return !isUrlExcluded(url, exclude) && patterns.some((pattern) => matchesGlobPattern(url, pattern));
-				});
+			const filteredUrls = filterSitemapUrls(sitemapUrls, hasGlobPatterns, exclude, patterns, allowSubdomains);
+			if (hasGlobPatterns) {
+				startingUrls = filteredUrls;
+				progress.sitemap.found = sitemapUrls.length;
+				progress.sitemap.processed = filteredUrls.length;
+				onProgress?.(progress);
+				mainSitemapProcessed = true;
+			} else if (filteredUrls.length > 0) {
 				startingUrls = filteredUrls;
 				progress.sitemap.found = sitemapUrls.length;
 				progress.sitemap.processed = filteredUrls.length;
 				onProgress?.(progress);
 				mainSitemapProcessed = true;
-			} else {
-				const filteredUrls = sitemapUrls.filter((url) => {
-					return !isUrlExcluded(url, exclude);
-				});
-				if (filteredUrls.length > 0) {
-					startingUrls = filteredUrls;
-					progress.sitemap.found = sitemapUrls.length;
-					progress.sitemap.processed = filteredUrls.length;
-					onProgress?.(progress);
-					mainSitemapProcessed = true;
-				}
 			}
 		} catch (error) {
 			sitemapAttempts.push({
@@ -348,31 +358,24 @@ async function crawlAndGenerate(options, onProgress) {
 					`${baseUrl}/sitemap-index.xml`
 				];
 				for (const sitemapUrl of commonSitemaps) try {
-					const altUrls = await loadSitemapWithoutRetries(sitemapUrl);
+					const altUrls = await loadSitemap(sitemapUrl);
 					sitemapAttempts.push({
 						url: sitemapUrl,
 						success: true
 					});
-					if (patterns.some((p) => p.isGlob)) {
-						const filteredUrls = altUrls.filter((url) => {
-							return !isUrlExcluded(url, exclude) && patterns.some((pattern) => matchesGlobPattern(url, pattern));
-						});
+					const filteredUrls = filterSitemapUrls(altUrls, hasGlobPatterns, exclude, patterns, allowSubdomains);
+					if (hasGlobPatterns) {
+						startingUrls = filteredUrls;
+						progress.sitemap.found = altUrls.length;
+						progress.sitemap.processed = filteredUrls.length;
+						onProgress?.(progress);
+						break;
+					} else if (filteredUrls.length > 0) {
 						startingUrls = filteredUrls;
 						progress.sitemap.found = altUrls.length;
 						progress.sitemap.processed = filteredUrls.length;
 						onProgress?.(progress);
 						break;
-					} else {
-						const filteredUrls = altUrls.filter((url) => {
-							return !isUrlExcluded(url, exclude);
-						});
-						if (filteredUrls.length > 0) {
-							startingUrls = filteredUrls;
-							progress.sitemap.found = altUrls.length;
-							progress.sitemap.processed = filteredUrls.length;
-							onProgress?.(progress);
-							break;
-						}
 					}
 				} catch (error) {
 					sitemapAttempts.push({
@@ -398,169 +401,256 @@ async function crawlAndGenerate(options, onProgress) {
 		progress.sitemap.status = "completed";
 		progress.crawling.total = startingUrls.length;
 		onProgress?.(progress);
-	} else if (skipSitemap && startingUrls.length > 0) {
+	} else if ((skipSitemap || singlePageMode) && startingUrls.length > 0) {
 		progress.sitemap.status = "completed";
 		progress.sitemap.found = 0;
 		progress.sitemap.processed = 0;
 		progress.crawling.total = startingUrls.length;
 		onProgress?.(progress);
 	}
-	if (!existsSync(outputDir)) mkdirSync(outputDir, { recursive: true });
+	mkdirSync(outputDir, { recursive: true });
 	const results = [];
 	const processedUrls = /* @__PURE__ */ new Set();
+	const allowedRegistrableDomains = allowSubdomains ? new Set(startingUrls.map((u) => {
+		try {
+			return getRegistrableDomain(new URL(u).hostname);
+		} catch {
+			return "";
+		}
+	}).filter(Boolean)) : void 0;
 	const shouldCrawlUrl = (url) => {
-		if (isUrlExcluded(url, exclude)) return false;
-		if (!patterns.some((p) => p.isGlob)) return true;
-		return patterns.some((pattern) => matchesGlobPattern(url, pattern));
-	};
-	const createRequestHandler = (crawlerType) => {
-		return async ({ request, body, page, enqueueLinks, response }) => {
-			const startTime = Date.now();
-			progress.crawling.currentUrl = request.loadedUrl;
-			onProgress?.(progress);
-			if (response?.statusCode && (response.statusCode < 200 || response.statusCode >= 300)) return;
-			const homePageUrl = new URL(startingUrls[0]).origin;
-			let html;
-			let title;
-			if (crawlerType === "playwright") {
-				await page.waitForLoadState("networkidle");
-				title = await page.title();
-				html = await page.innerHTML("html");
-			} else {
-				html = typeof body === "string" ? body : body.toString();
-				title = "";
+		if (isUrlExcluded(url, exclude, allowSubdomains)) return false;
+		if (!hasGlobPatterns) {
+			if (allowedRegistrableDomains) try {
+				return allowedRegistrableDomains.has(getRegistrableDomain(new URL(url).hostname));
+			} catch {
+				return false;
 			}
-			const metadata = extractMetadata(html, request.loadedUrl);
-			if (!title) title = metadata.title;
-			const shouldProcessMarkdown = shouldCrawlUrl(request.loadedUrl);
-			const pageOrigin = origin || new URL(request.loadedUrl).origin;
-			if (onPage && shouldProcessMarkdown) await onPage({
-				url: request.loadedUrl,
-				html,
+			return true;
+		}
+		return patterns.some((pattern) => matchesGlobPattern(url, pattern, allowSubdomains));
+	};
+	const recordLatency = (ms) => {
+		const lat = progress.crawling.latency;
+		lat.total += ms;
+		lat.count++;
+		if (ms < lat.min) lat.min = ms;
+		if (ms > lat.max) lat.max = ms;
+	};
+	const normalizedHomePageUrl = (startingUrls.length > 0 ? new URL(startingUrls[0]).origin : "").replace(URL_TRAILING_SLASH_RE, "");
+	const createdDirs = /* @__PURE__ */ new Set();
+	const sharedOrigin = origin || "";
+	const processPage = async (url, content, initialTitle, depth, isMarkdown = false) => {
+		const parsedUrl = new URL(url);
+		const shouldProcessMarkdown = shouldCrawlUrl(url);
+		const pageOrigin = sharedOrigin || parsedUrl.origin;
+		let md;
+		let metadata;
+		if (isMarkdown) {
+			md = content;
+			metadata = {
+				title: initialTitle || parsedUrl.pathname,
+				links: []
+			};
+		} else {
+			const { extraction, getMetadata } = extractMetadataInline(parsedUrl, allowedRegistrableDomains);
+			md = htmlToMarkdown(content, {
+				origin: pageOrigin,
+				extraction
+			});
+			metadata = getMetadata();
+		}
+		let title = initialTitle || metadata.title;
+		if (shouldProcessMarkdown) {
+			const pageData = {
+				url,
+				html: isMarkdown ? "" : content,
 				title,
 				metadata,
 				origin: pageOrigin
-			});
-			let md = "";
-			if (shouldProcessMarkdown) md = htmlToMarkdown(html, { origin: pageOrigin });
-			let filePath;
-			if (shouldProcessMarkdown && generateIndividualMd) {
-				const urlObj = new URL(request.loadedUrl);
-				const safeSegments = (urlObj.pathname === "/" ? "/index" : urlObj.pathname).replace(URL_TRAILING_SLASH_RE, "").split("/").filter((seg) => seg.length > 0).map((seg) => seg.replace(URL_PATH_UNSAFE_CHARS_RE, "-"));
-				filePath = join(outputDir, normalize(`${safeSegments.length > 0 ? safeSegments.join("/") : "index"}.md`));
-				const fileDir = dirname(filePath);
-				if (fileDir && !existsSync(fileDir)) mkdirSync(fileDir, { recursive: true });
-				await writeFile(filePath, md, "utf-8");
-			}
-			const isHomePage = request.loadedUrl.replace(URL_TRAILING_SLASH_RE, "") === homePageUrl.replace(URL_TRAILING_SLASH_RE, "");
-			if (shouldProcessMarkdown || isHomePage) {
-				const result = {
-					url: request.loadedUrl,
-					title,
-					content: md,
-					filePath: shouldProcessMarkdown ? filePath : void 0,
-					timestamp: startTime,
-					success: true,
-					metadata,
-					depth: request.userData?.depth || 0
-				};
-				results.push(result);
-				progress.crawling.processed = results.length;
-				onProgress?.(progress);
-			}
-			if (followLinks && (request.userData?.depth || 0) < maxDepth) {
-				const currentDepth = (request.userData?.depth || 0) + 1;
-				const filteredLinks = metadata.links.filter((link) => {
-					return shouldCrawlUrl(link);
-				});
-				if (enqueueLinks) await enqueueLinks({
-					urls: filteredLinks,
-					userData: { depth: currentDepth }
-				});
-				else for (const link of filteredLinks) if (!processedUrls.has(link)) processedUrls.add(link);
+			};
+			await hooks.callHook("crawl:page", pageData);
+			title = pageData.title;
+		}
+		let filePath;
+		if (shouldProcessMarkdown && generateIndividualMd) {
+			const urlPath = parsedUrl.pathname === "/" ? "/index" : parsedUrl.pathname;
+			const hostPrefix = allowSubdomains ? [parsedUrl.hostname.replace(URL_PATH_UNSAFE_CHARS_RE, "-")] : [];
+			const pathSegments = urlPath.replace(URL_TRAILING_SLASH_RE, "").split("/").filter((seg) => seg.length > 0);
+			const safeSegments = [...hostPrefix, ...pathSegments.map((seg) => seg.replace(URL_PATH_UNSAFE_CHARS_RE, "-"))];
+			filePath = join(outputDir, normalize(`${safeSegments.length > 0 ? safeSegments.join("/") : "index"}.md`));
+			const contentCtx = {
+				url,
+				title,
+				content: md,
+				filePath
+			};
+			await hooks.callHook("crawl:content", contentCtx);
+			md = contentCtx.content;
+			filePath = contentCtx.filePath;
+			const fileDir = dirname(filePath);
+			if (fileDir && !createdDirs.has(fileDir)) {
+				await mkdir(fileDir, { recursive: true });
+				createdDirs.add(fileDir);
 			}
-		};
+			await writeFile(filePath, md, "utf-8");
+		}
+		const isHomePage = parsedUrl.pathname === "/" && parsedUrl.origin === normalizedHomePageUrl;
+		if (shouldProcessMarkdown || isHomePage) {
+			const result = {
+				url,
+				title,
+				content: md,
+				filePath: shouldProcessMarkdown ? filePath : void 0,
+				timestamp: Date.now(),
+				success: true,
+				metadata,
+				depth
+			};
+			results.push(result);
+			progress.crawling.processed = results.length;
+			onProgress?.(progress);
+		}
+		if (followLinks && !singlePageMode && depth < maxDepth) {
+			const filteredLinks = metadata.links.filter((link) => shouldCrawlUrl(link));
+			for (const link of filteredLinks) processedUrls.add(link);
+		}
 	};
-	let crawler;
-	const crawlerOptions = {
-		requestHandler: createRequestHandler(driver),
-		errorHandler: async ({ request, response, error }) => {
-			if (verbose) console.error(`[ERROR] URL: ${request.url}, Status: ${response?.statusCode || "N/A"}, Error: ${error?.message || "Unknown"}`);
-			if (response?.statusCode && response?.statusCode >= 400) {
-				request.noRetry = true;
-				const result = {
-					url: request.url,
-					title: "",
-					content: "",
-					timestamp: Date.now(),
-					success: false,
-					error: `HTTP ${response.statusCode}`,
-					metadata: {
-						title: "",
-						description: "",
-						links: []
-					},
-					depth: request.userData?.depth || 0
+	const urlsToProcess = startingUrls.slice(0, maxRequestsPerCrawl);
+	progress.crawling.status = "processing";
+	progress.crawling.total = urlsToProcess.length;
+	onProgress?.(progress);
+	if (driver === "playwright") {
+		const { log, PlaywrightCrawler, purgeDefaultStorages } = await import("crawlee");
+		if (verbose) log.setLevel(log.LEVELS.INFO);
+		else log.setLevel(log.LEVELS.OFF);
+		const crawlerOptions = {
+			requestHandler: async ({ request, page }) => {
+				progress.crawling.currentUrl = request.loadedUrl;
+				onProgress?.(progress);
+				const urlCtx = {
+					url: request.loadedUrl,
+					skip: false
 				};
-				results.push(result);
-			} else if (error) {
+				await hooks.callHook("crawl:url", urlCtx);
+				if (urlCtx.skip) return;
+				const fetchStart = Date.now();
+				await page.waitForLoadState("networkidle");
+				const title = await page.title();
+				const html = await page.innerHTML("html");
+				recordLatency(Date.now() - fetchStart);
+				await processPage(request.loadedUrl, html, title, request.userData?.depth || 0);
+			},
+			errorHandler: async ({ request, response, error }) => {
+				if (verbose) console.error(`[ERROR] URL: ${request.url}, Status: ${response?.statusCode || "N/A"}, Error: ${error?.message || "Unknown"}`);
 				request.noRetry = true;
-				const result = {
+				progress.crawling.failed++;
+				results.push({
 					url: request.url,
 					title: "",
 					content: "",
 					timestamp: Date.now(),
 					success: false,
-					error: error.message || "Unknown error",
+					error: response?.statusCode ? `HTTP ${response.statusCode}` : error?.message || "Unknown error",
 					metadata: {
 						title: "",
 						description: "",
 						links: []
 					},
 					depth: request.userData?.depth || 0
-				};
-				results.push(result);
-			}
-		},
-		maxRequestsPerCrawl,
-		respectRobotsTxtFile: false
-	};
-	if (crawlDelay) crawlerOptions.requestHandlerTimeoutSecs = crawlDelay;
-	if (driver === "playwright") {
-		const playwrightOptions = crawlerOptions;
-		if (useChrome) playwrightOptions.launchContext = {
-			...playwrightOptions.launchContext,
+				});
+			},
+			maxRequestsPerCrawl,
+			respectRobotsTxtFile: false
+		};
+		if (crawlDelay) crawlerOptions.requestHandlerTimeoutSecs = crawlDelay;
+		if (useChrome) crawlerOptions.launchContext = {
+			...crawlerOptions.launchContext,
 			useChrome
 		};
-		crawler = new PlaywrightCrawler(playwrightOptions);
-	} else crawler = new HttpCrawler(crawlerOptions);
-	const initialRequests = startingUrls.map((url) => ({
-		url,
-		userData: { depth: 0 }
-	}));
-	progress.crawling.status = "processing";
-	progress.crawling.total = startingUrls.length;
-	onProgress?.(progress);
-	try {
-		await crawler.run(initialRequests);
-	} catch (error) {
-		if (verbose) {
-			console.error(`[CRAWLER ERROR] ${error instanceof Error ? error.message : "Unknown error"}`);
-			console.error(`[CRAWLER ERROR] Stack trace:`, error instanceof Error ? error.stack : "No stack trace");
+		const crawler = new PlaywrightCrawler(crawlerOptions);
+		const initialRequests = urlsToProcess.map((url) => ({
+			url,
+			userData: { depth: 0 }
+		}));
+		try {
+			await crawler.run(initialRequests);
+		} catch (error) {
+			const msg = error instanceof Error ? error.message : "";
+			if (msg.includes("wmic") || msg.includes("ENOENT")) throw new Error(`Crawlee failed to spawn a system process (${msg}). On Windows 11+, wmic.exe is no longer available. Upgrade crawlee to >=3.16.0 or use the HTTP driver instead (--driver http).`);
+			if (verbose) {
+				console.error(`[CRAWLER ERROR] ${msg || "Unknown error"}`);
+				console.error(`[CRAWLER ERROR] Stack trace:`, error instanceof Error ? error.stack : "No stack trace");
+			}
+			throw error;
 		}
-		throw error;
-	}
+		await purgeDefaultStorages();
+	} else await runConcurrent(urlsToProcess, DEFAULT_CONCURRENCY, async (url) => {
+		progress.crawling.currentUrl = url;
+		onProgress?.(progress);
+		if (crawlDelay) {
+			const delay = crawlDelay;
+			await new Promise((resolve) => setTimeout(resolve, delay * 1e3));
+		}
+		const urlCtx = {
+			url,
+			skip: false
+		};
+		await hooks.callHook("crawl:url", urlCtx);
+		if (urlCtx.skip) return;
+		try {
+			const fetchStart = Date.now();
+			const response = await ofetch.raw(url, {
+				headers: FETCH_HEADERS,
+				responseType: "text",
+				retry: 2,
+				retryDelay: 500,
+				timeout: 1e4,
+				onResponseError({ response }) {
+					if (response.status === 429) {
+						const retryAfter = response.headers.get("retry-after");
+						const delaySec = retryAfter ? Number.parseInt(retryAfter) || 1 : 2;
+						if (!crawlDelay || delaySec > crawlDelay) crawlDelay = delaySec;
+					}
+				}
+			});
+			recordLatency(Date.now() - fetchStart);
+			const body = response._data ?? "";
+			const contentType = response.headers.get("content-type") || "";
+			await processPage(url, body, "", 0, contentType.includes("text/markdown") || contentType.includes("text/x-markdown"));
+		} catch (error) {
+			if (verbose) console.error(`[ERROR] URL: ${url}, Error: ${error instanceof Error ? error.message : "Unknown"}`);
+			progress.crawling.failed++;
+			results.push({
+				url,
+				title: "",
+				content: "",
+				timestamp: Date.now(),
+				success: false,
+				error: error instanceof Error ? error.message : "Unknown error",
+				metadata: {
+					title: "",
+					description: "",
+					links: []
+				},
+				depth: 0
+			});
+			progress.crawling.processed = results.length;
+			onProgress?.(progress);
+		}
+	});
 	progress.crawling.status = "completed";
 	onProgress?.(progress);
+	await hooks.callHook("crawl:done", { results });
 	if (results.some((r) => r.success)) {
 		progress.generation.status = "generating";
 		onProgress?.(progress);
 		const successfulResults = results.filter((r) => r.success);
 		const firstUrl = new URL(withHttps(urls[0]));
-		const origin = firstUrl.origin;
+		const originUrl = firstUrl.origin;
 		const homePageResult = successfulResults.find((r) => {
 			const resultUrl = new URL(withHttps(r.url));
-			return resultUrl.href === origin || resultUrl.href === `${origin}/`;
+			return resultUrl.href === originUrl || resultUrl.href === `${originUrl}/`;
 		});
 		const siteName = siteNameOverride || homePageResult?.metadata?.title || homePageResult?.title || firstUrl.hostname;
 		const description = descriptionOverride || homePageResult?.metadata?.description || successfulResults[0]?.metadata?.description;
@@ -586,7 +676,7 @@ async function crawlAndGenerate(options, onProgress) {
 				})),
 				siteName,
 				description,
-				origin: origin || firstUrl.origin,
+				origin: originUrl || firstUrl.origin,
 				generateFull: generateLlmsFullTxt,
 				outputDir
 			});
@@ -604,7 +694,6 @@ async function crawlAndGenerate(options, onProgress) {
 		progress.generation.status = "completed";
 		onProgress?.(progress);
 	}
-	await purgeDefaultStorages();
 	return results;
 }
 //#endregion