@mdream/crawl 1.0.1 → 1.0.3

package/dist/_chunks/crawl.mjs

@@ -136,6 +136,20 @@ const FETCH_HEADERS = {
 	"Accept": "text/html,application/xhtml+xml,text/markdown"
 };
 const DEFAULT_CONCURRENCY = 20;
+const IGNORED_PATH_PREFIXES = [
+	"/cdn-cgi/",
+	"/_next/",
+	"/_nuxt/",
+	"/__",
+	"/wp-admin/",
+	"/wp-json/",
+	"/wp-includes/",
+	"/wp-content/uploads/",
+	"/api/",
+	"/assets/",
+	"/static/"
+];
+const HTML_EXTENSIONS_RE = /\.(html?|php|aspx?|jsp)$/i;
 function extractCdataUrl(url) {
 	if (url.startsWith("<![CDATA[") && url.endsWith("]]>")) return url.slice(9, -3);
 	return url;
@@ -186,7 +200,8 @@ function extractMetadataInline(parsedUrl, allowedDomains) {
 				const href = el.attributes.href;
 				if (href) try {
 					const resolved = new URL(href, url);
-					const absoluteUrl = resolved.href;
+					resolved.hash = "";
+					const absoluteUrl = resolved.href.replace(URL_TRAILING_SLASH_RE, "") || resolved.href;
 					if (allowedDomains) {
 						const domain = getRegistrableDomain(resolved.hostname);
 						if (domain && allowedDomains.has(domain)) links.add(absoluteUrl);
@@ -418,6 +433,16 @@ async function crawlAndGenerate(options, onProgress) {
 			return "";
 		}
 	}).filter(Boolean)) : void 0;
+	const isContentUrl = (url) => {
+		try {
+			const pathname = new URL(url).pathname;
+			for (let i = 0; i < IGNORED_PATH_PREFIXES.length; i++) if (pathname.startsWith(IGNORED_PATH_PREFIXES[i])) return false;
+			if (pathname.lastIndexOf(".") > pathname.lastIndexOf("/")) return HTML_EXTENSIONS_RE.test(pathname);
+			return true;
+		} catch {
+			return false;
+		}
+	};
 	const shouldCrawlUrl = (url) => {
 		if (isUrlExcluded(url, exclude, allowSubdomains)) return false;
 		if (!hasGlobPatterns) {
@@ -512,11 +537,20 @@ async function crawlAndGenerate(options, onProgress) {
 			onProgress?.(progress);
 		}
 		if (followLinks && !singlePageMode && depth < maxDepth) {
-			const filteredLinks = metadata.links.filter((link) => shouldCrawlUrl(link));
-			for (const link of filteredLinks) processedUrls.add(link);
+			const filteredLinks = metadata.links.filter((link) => shouldCrawlUrl(link) && isContentUrl(link));
+			for (const link of filteredLinks) if (!processedUrls.has(link)) {
+				processedUrls.add(link);
+				pendingUrls.push({
+					url: link,
+					depth: depth + 1
+				});
+			}
 		}
 	};
+	const pendingUrls = [];
 	const urlsToProcess = startingUrls.slice(0, maxRequestsPerCrawl);
+	for (const url of urlsToProcess) processedUrls.add(url.replace(URL_TRAILING_SLASH_RE, "") || url);
+	let totalProcessed = 0;
 	progress.crawling.status = "processing";
 	progress.crawling.total = urlsToProcess.length;
 	onProgress?.(progress);
@@ -569,12 +603,24 @@ async function crawlAndGenerate(options, onProgress) {
 			useChrome
 		};
 		const crawler = new PlaywrightCrawler(crawlerOptions);
-		const initialRequests = urlsToProcess.map((url) => ({
+		const allRequests = [...urlsToProcess.map((url) => ({
 			url,
 			userData: { depth: 0 }
-		}));
+		}))];
 		try {
-			await crawler.run(initialRequests);
+			await crawler.run(allRequests);
+			totalProcessed += urlsToProcess.length;
+			while (pendingUrls.length > 0 && totalProcessed < maxRequestsPerCrawl) {
+				const batch = pendingUrls.splice(0, maxRequestsPerCrawl - totalProcessed);
+				progress.crawling.total += batch.length;
+				onProgress?.(progress);
+				const batchRequests = batch.map((item) => ({
+					url: item.url,
+					userData: { depth: item.depth }
+				}));
+				await crawler.run(batchRequests);
+				totalProcessed += batch.length;
+			}
 		} catch (error) {
 			const msg = error instanceof Error ? error.message : "";
 			if (msg.includes("wmic") || msg.includes("ENOENT")) throw new Error(`Crawlee failed to spawn a system process (${msg}). On Windows 11+, wmic.exe is no longer available. Upgrade crawlee to >=3.16.0 or use the HTTP driver instead (--driver http).`);
@@ -585,60 +631,74 @@ async function crawlAndGenerate(options, onProgress) {
 			throw error;
 		}
 		await purgeDefaultStorages();
-	} else await runConcurrent(urlsToProcess, DEFAULT_CONCURRENCY, async (url) => {
-		progress.crawling.currentUrl = url;
-		onProgress?.(progress);
-		if (crawlDelay) {
-			const delay = crawlDelay;
-			await new Promise((resolve) => setTimeout(resolve, delay * 1e3));
-		}
-		const urlCtx = {
-			url,
-			skip: false
-		};
-		await hooks.callHook("crawl:url", urlCtx);
-		if (urlCtx.skip) return;
-		try {
-			const fetchStart = Date.now();
-			const response = await ofetch.raw(url, {
-				headers: FETCH_HEADERS,
-				responseType: "text",
-				retry: 2,
-				retryDelay: 500,
-				timeout: 1e4,
-				onResponseError({ response }) {
-					if (response.status === 429) {
-						const retryAfter = response.headers.get("retry-after");
-						const delaySec = retryAfter ? Number.parseInt(retryAfter) || 1 : 2;
-						if (!crawlDelay || delaySec > crawlDelay) crawlDelay = delaySec;
-					}
-				}
-			});
-			recordLatency(Date.now() - fetchStart);
-			const body = response._data ?? "";
-			const contentType = response.headers.get("content-type") || "";
-			await processPage(url, body, "", 0, contentType.includes("text/markdown") || contentType.includes("text/x-markdown"));
-		} catch (error) {
-			if (verbose) console.error(`[ERROR] URL: ${url}, Error: ${error instanceof Error ? error.message : "Unknown"}`);
-			progress.crawling.failed++;
-			results.push({
+	} else {
+		const fetchPage = async (url, depth) => {
+			progress.crawling.currentUrl = url;
+			onProgress?.(progress);
+			if (crawlDelay) {
+				const delay = crawlDelay;
+				await new Promise((resolve) => setTimeout(resolve, delay * 1e3));
+			}
+			const urlCtx = {
 				url,
-				title: "",
-				content: "",
-				timestamp: Date.now(),
-				success: false,
-				error: error instanceof Error ? error.message : "Unknown error",
-				metadata: {
+				skip: false
+			};
+			await hooks.callHook("crawl:url", urlCtx);
+			if (urlCtx.skip) return;
+			try {
+				const fetchStart = Date.now();
+				const response = await ofetch.raw(url, {
+					headers: FETCH_HEADERS,
+					responseType: "text",
+					retry: 2,
+					retryDelay: 500,
+					timeout: 1e4,
+					onResponseError({ response }) {
+						if (response.status === 429) {
+							const retryAfter = response.headers.get("retry-after");
+							const delaySec = retryAfter ? Number.parseInt(retryAfter) || 1 : 2;
+							if (!crawlDelay || delaySec > crawlDelay) crawlDelay = delaySec;
+						}
+					}
+				});
+				recordLatency(Date.now() - fetchStart);
+				const body = response._data ?? "";
+				const contentType = response.headers.get("content-type") || "";
+				const isHtml = contentType.includes("text/html") || contentType.includes("application/xhtml");
+				const isMarkdown = contentType.includes("text/markdown") || contentType.includes("text/x-markdown");
+				if (!isHtml && !isMarkdown) return;
+				await processPage(url, body, "", depth, isMarkdown);
+			} catch (error) {
+				if (verbose) console.error(`[ERROR] URL: ${url}, Error: ${error instanceof Error ? error.message : "Unknown"}`);
+				progress.crawling.failed++;
+				results.push({
+					url,
 					title: "",
-					description: "",
-					links: []
-				},
-				depth: 0
-			});
-			progress.crawling.processed = results.length;
+					content: "",
+					timestamp: Date.now(),
+					success: false,
+					error: error instanceof Error ? error.message : "Unknown error",
+					metadata: {
+						title: "",
+						description: "",
+						links: []
+					},
+					depth
+				});
+				progress.crawling.processed = results.length;
+				onProgress?.(progress);
+			}
+		};
+		await runConcurrent(urlsToProcess, DEFAULT_CONCURRENCY, (url) => fetchPage(url, 0));
+		totalProcessed += urlsToProcess.length;
+		while (pendingUrls.length > 0 && totalProcessed < maxRequestsPerCrawl) {
+			const batch = pendingUrls.splice(0, maxRequestsPerCrawl - totalProcessed);
+			progress.crawling.total += batch.length;
 			onProgress?.(progress);
+			await runConcurrent(batch, DEFAULT_CONCURRENCY, (item) => fetchPage(item.url, item.depth));
+			totalProcessed += batch.length;
 		}
-	});
+	}
 	progress.crawling.status = "completed";
 	onProgress?.(progress);
 	await hooks.callHook("crawl:done", { results });

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@mdream/crawl",
   "type": "module",
-  "version": "1.0.1",
+  "version": "1.0.3",
   "description": "Mdream Crawl generates comprehensive llms.txt artifacts from a single URL, using mdream to convert HTML to Markdown.",
   "author": {
     "name": "Harlan Wilton",
@@ -55,16 +55,16 @@
   },
   "dependencies": {
     "@clack/prompts": "^1.1.0",
-    "c12": "^3.0.4",
-    "hookable": "^5.5.3",
+    "c12": "^3.3.3",
+    "hookable": "^6.1.0",
     "nypm": "^0.6.5",
     "ofetch": "^1.5.1",
     "pathe": "^2.0.3",
     "picomatch": "^4.0.3",
     "tldts": "^7.0.26",
     "ufo": "^1.6.3",
-    "@mdream/js": "1.0.1",
-    "mdream": "1.0.1"
+    "@mdream/js": "1.0.3",
+    "mdream": "1.0.3"
   },
   "devDependencies": {
     "@types/picomatch": "^4.0.2"