@mdream/crawl 0.7.1 → 0.8.0

package/dist/_chunks/{crawl-B5MaCj6O.mjs → crawl-BwURA9nQ.mjs}

@@ -1,6 +1,7 @@
 import { existsSync, mkdirSync } from "node:fs";
 import { writeFile } from "node:fs/promises";
-import { HttpCrawler, Sitemap, purgeDefaultStorages } from "crawlee";
+import * as p from "@clack/prompts";
+import { HttpCrawler, PlaywrightCrawler, log, purgeDefaultStorages } from "crawlee";
 import { generateLlmsTxtArtifacts, htmlToMarkdown } from "mdream";
 import { withMinimalPreset } from "mdream/preset/minimal";
 import { dirname, join, normalize, resolve } from "pathe";
@@ -46,7 +47,7 @@ function parseUrlPattern(input) {
 			pattern,
 			isGlob: true
 		};
-	} catch (error) {
+	} catch {
 		throw new Error(`Invalid URL pattern: "${input}". Please provide a valid URL with glob patterns (e.g., "example.com/docs/*" or "https://example.com/api/**").`);
 	}
 }
@@ -178,9 +179,25 @@ function extractMetadata(html, url) {
 
 //#endregion
 //#region src/crawl.ts
+async function loadSitemapWithoutRetries(sitemapUrl) {
+	const response = await fetch(sitemapUrl);
+	if (!response.ok) throw new Error(`Sitemap not found: ${response.status}`);
+	const xmlContent = await response.text();
+	const urls = [];
+	const urlRegex = /<loc>(.*?)<\/loc>/g;
+	let match;
+	while (true) {
+		match = urlRegex.exec(xmlContent);
+		if (match === null) break;
+		urls.push(match[1]);
+	}
+	return urls;
+}
 async function crawlAndGenerate(options, onProgress) {
-	const { urls, outputDir: rawOutputDir, maxRequestsPerCrawl = Number.MAX_SAFE_INTEGER, generateLlmsTxt = true, generateLlmsFullTxt = false, generateIndividualMd = true, origin, driver = "http", followLinks = false, maxDepth = 1, globPatterns = [], crawlDelay, exclude = [], siteNameOverride, descriptionOverride } = options;
+	const { urls, outputDir: rawOutputDir, maxRequestsPerCrawl = Number.MAX_SAFE_INTEGER, generateLlmsTxt = true, generateLlmsFullTxt = false, generateIndividualMd = true, origin, driver = "http", followLinks = false, maxDepth = 1, globPatterns = [], crawlDelay, exclude = [], siteNameOverride, descriptionOverride, verbose = false } = options;
 	const outputDir = resolve(normalize(rawOutputDir));
+	if (verbose) log.setLevel(log.LEVELS.INFO);
+	else log.setLevel(log.LEVELS.OFF);
 	let patterns;
 	try {
 		patterns = globPatterns.length > 0 ? globPatterns : urls.map(parseUrlPattern);
@@ -201,6 +218,7 @@ async function crawlAndGenerate(options, onProgress) {
 		},
 		generation: { status: "idle" }
 	};
+	const sitemapAttempts = [];
 	if (startingUrls.length > 0) {
 		const baseUrl = new URL(startingUrls[0]).origin;
 		const homePageUrl = baseUrl;
@@ -216,8 +234,12 @@ async function crawlAndGenerate(options, onProgress) {
 				onProgress?.(progress);
 				const robotsSitemaps = sitemapMatches.map((match) => match.replace(/Sitemap:\s*/i, "").trim());
 				for (const sitemapUrl of robotsSitemaps) try {
-					const { urls: robotsUrls } = await Sitemap.load(sitemapUrl);
-					const hasGlobPatterns = patterns.some((p) => p.isGlob);
+					const robotsUrls = await loadSitemapWithoutRetries(sitemapUrl);
+					sitemapAttempts.push({
+						url: sitemapUrl,
+						success: true
+					});
+					const hasGlobPatterns = patterns.some((p$1) => p$1.isGlob);
 					if (hasGlobPatterns) {
 						const filteredUrls = robotsUrls.filter((url) => {
 							return !isUrlExcluded(url, exclude) && patterns.some((pattern) => matchesGlobPattern(url, pattern));
@@ -237,15 +259,24 @@ async function crawlAndGenerate(options, onProgress) {
 							break;
 						}
 					}
-				} catch {
-					continue;
+				} catch (error) {
+					sitemapAttempts.push({
+						url: sitemapUrl,
+						success: false,
+						error: error instanceof Error ? error.message : "Unknown error"
+					});
 				}
 			}
 		}
 		let mainSitemapProcessed = false;
+		const mainSitemapUrl = `${baseUrl}/sitemap.xml`;
 		try {
-			const { urls: sitemapUrls } = await Sitemap.load(`${baseUrl}/sitemap.xml`);
-			const hasGlobPatterns = patterns.some((p) => p.isGlob);
+			const sitemapUrls = await loadSitemapWithoutRetries(mainSitemapUrl);
+			sitemapAttempts.push({
+				url: mainSitemapUrl,
+				success: true
+			});
+			const hasGlobPatterns = patterns.some((p$1) => p$1.isGlob);
 			if (hasGlobPatterns) {
 				const filteredUrls = sitemapUrls.filter((url) => {
 					return !isUrlExcluded(url, exclude) && patterns.some((pattern) => matchesGlobPattern(url, pattern));
@@ -267,7 +298,12 @@ async function crawlAndGenerate(options, onProgress) {
 					mainSitemapProcessed = true;
 				}
 			}
-		} catch {
+		} catch (error) {
+			sitemapAttempts.push({
+				url: mainSitemapUrl,
+				success: false,
+				error: error instanceof Error ? error.message : "Unknown error"
+			});
 			if (!mainSitemapProcessed) {
 				const commonSitemaps = [
 					`${baseUrl}/sitemap_index.xml`,
@@ -275,8 +311,12 @@ async function crawlAndGenerate(options, onProgress) {
 					`${baseUrl}/sitemap-index.xml`
 				];
 				for (const sitemapUrl of commonSitemaps) try {
-					const { urls: altUrls } = await Sitemap.load(sitemapUrl);
-					const hasGlobPatterns = patterns.some((p) => p.isGlob);
+					const altUrls = await loadSitemapWithoutRetries(sitemapUrl);
+					sitemapAttempts.push({
+						url: sitemapUrl,
+						success: true
+					});
+					const hasGlobPatterns = patterns.some((p$1) => p$1.isGlob);
 					if (hasGlobPatterns) {
 						const filteredUrls = altUrls.filter((url) => {
 							return !isUrlExcluded(url, exclude) && patterns.some((pattern) => matchesGlobPattern(url, pattern));
@@ -298,11 +338,26 @@ async function crawlAndGenerate(options, onProgress) {
 							break;
 						}
 					}
-				} catch {
-					continue;
+				} catch (error$1) {
+					sitemapAttempts.push({
+						url: sitemapUrl,
+						success: false,
+						error: error$1 instanceof Error ? error$1.message : "Unknown error"
+					});
 				}
 			}
 		}
+		const successfulSitemaps = sitemapAttempts.filter((a) => a.success);
+		const failedSitemaps = sitemapAttempts.filter((a) => !a.success);
+		if (successfulSitemaps.length > 0) {
+			const sitemapUrl = successfulSitemaps[0].url;
+			if (progress.sitemap.processed > 0) p.note(`Found sitemap at ${sitemapUrl} with ${progress.sitemap.processed} URLs`, "Sitemap Discovery");
+			else p.note(`Found sitemap at ${sitemapUrl} but no URLs matched your search criteria`, "Sitemap Discovery");
+		} else if (failedSitemaps.length > 0) {
+			const firstAttempt = failedSitemaps[0];
+			if (firstAttempt.error?.includes("404")) p.note(`No sitemap found, using crawler to discover pages`, "Sitemap Discovery");
+			else p.note(`Could not access sitemap: ${firstAttempt.error}`, "Sitemap Discovery");
+		}
 		if (!startingUrls.includes(homePageUrl)) startingUrls.unshift(homePageUrl);
 		progress.sitemap.status = "completed";
 		progress.crawling.total = startingUrls.length;
@@ -313,16 +368,16 @@ async function crawlAndGenerate(options, onProgress) {
 	const processedUrls = new Set();
 	const shouldCrawlUrl = (url) => {
 		if (isUrlExcluded(url, exclude)) return false;
-		if (!patterns.some((p) => p.isGlob)) return true;
+		if (!patterns.some((p$1) => p$1.isGlob)) return true;
 		return patterns.some((pattern) => matchesGlobPattern(url, pattern));
 	};
 	const createRequestHandler = (crawlerType) => {
-		return async ({ request, body, page, enqueueLinks }) => {
+		return async ({ request, body, page, enqueueLinks, response }) => {
 			const startTime = Date.now();
 			progress.crawling.currentUrl = request.loadedUrl;
 			onProgress?.(progress);
-			const baseUrl = new URL(startingUrls[0]).origin;
-			const homePageUrl = baseUrl;
+			if (response?.statusCode && (response.statusCode < 200 || response.statusCode >= 300)) return;
+			const homePageUrl = new URL(startingUrls[0]).origin;
 			let html;
 			let title;
 			if (crawlerType === "playwright") {
@@ -346,20 +401,22 @@ async function crawlAndGenerate(options, onProgress) {
 				const safeSegments = pathSegments.map((seg) => seg.replace(/[^\w\-]/g, "-"));
 				const filename = safeSegments.length > 0 ? safeSegments.join("/") : "index";
 				const safeFilename = normalize(`${filename}.md`);
-				filePath = join(outputDir, "md", safeFilename);
+				filePath = join(outputDir, safeFilename);
 				if (generateIndividualMd) {
 					const fileDir = dirname(filePath);
 					if (fileDir && !existsSync(fileDir)) mkdirSync(fileDir, { recursive: true });
 					await writeFile(filePath, md, "utf-8");
 				}
 			}
-			const isHomePage = request.loadedUrl === homePageUrl;
+			const normalizedUrl = request.loadedUrl.replace(/\/$/, "");
+			const normalizedHomePageUrl = homePageUrl.replace(/\/$/, "");
+			const isHomePage = normalizedUrl === normalizedHomePageUrl;
 			if (shouldProcessMarkdown || isHomePage) {
 				const result = {
 					url: request.loadedUrl,
 					title,
 					content: md,
-					filePath: generateIndividualMd && shouldProcessMarkdown ? filePath : void 0,
+					filePath: shouldProcessMarkdown ? filePath : void 0,
 					timestamp: startTime,
 					success: true,
 					metadata,
@@ -385,14 +442,15 @@ async function crawlAndGenerate(options, onProgress) {
 	let crawler;
 	const crawlerOptions = {
 		requestHandler: createRequestHandler(driver),
+		errorHandler: async ({ request, response }) => {
+			if (response?.statusCode && response?.statusCode >= 400) request.noRetry = true;
+		},
 		maxRequestsPerCrawl,
 		respectRobotsTxtFile: true
 	};
-	if (crawlDelay) crawlerOptions.requestHandlerTimeoutMillis = crawlDelay * 1e3;
-	if (driver === "playwright") {
-		const { PlaywrightCrawler: PlaywrightCrawlerClass } = await import("crawlee");
-		crawler = new PlaywrightCrawlerClass(crawlerOptions);
-	} else crawler = new HttpCrawler(crawlerOptions);
+	if (crawlDelay) crawlerOptions.requestHandlerTimeoutSecs = crawlDelay;
+	if (driver === "playwright") crawler = new PlaywrightCrawler(crawlerOptions);
+	else crawler = new HttpCrawler(crawlerOptions);
 	const initialRequests = startingUrls.map((url) => ({
 		url,
 		userData: { depth: 0 }
@@ -408,18 +466,29 @@ async function crawlAndGenerate(options, onProgress) {
 		onProgress?.(progress);
 		const successfulResults = results.filter((r$1) => r$1.success);
 		const firstUrl = new URL(withHttps(urls[0]));
+		const origin$1 = firstUrl.origin;
 		const homePageResult = successfulResults.find((r$1) => {
 			const resultUrl = new URL(withHttps(r$1.url));
-			const homeUrl = new URL(withHttps(urls[0]));
-			return resultUrl.href === homeUrl.href;
+			return resultUrl.href === origin$1 || resultUrl.href === `${origin$1}/`;
 		});
-		const siteName = siteNameOverride || homePageResult?.metadata?.title || firstUrl.hostname;
+		const siteName = siteNameOverride || homePageResult?.metadata?.title || homePageResult?.title || firstUrl.hostname;
 		const description = descriptionOverride || homePageResult?.metadata?.description || successfulResults[0]?.metadata?.description;
 		if (generateLlmsTxt || generateLlmsFullTxt) {
 			progress.generation.current = "Generating llms.txt files";
 			onProgress?.(progress);
-			const contentResults = successfulResults.filter((result) => result.content && result.content.trim().length > 0);
-			const processedFiles = contentResults.map((result) => ({
+			const contentResults = successfulResults.filter((result) => {
+				if (!result.content) return false;
+				const trimmedContent = result.content.trim();
+				const contentWithoutFrontmatter = trimmedContent.replace(/^---\s*\n(?:.*\n)*?---\s*/, "").trim();
+				return contentWithoutFrontmatter.length > 10;
+			});
+			const seenUrls = new Set();
+			const deduplicatedResults = contentResults.filter((result) => {
+				if (seenUrls.has(result.url)) return false;
+				seenUrls.add(result.url);
+				return true;
+			});
+			const processedFiles = deduplicatedResults.map((result) => ({
 				filePath: result.filePath,
 				title: result.title,
 				content: result.content,
@@ -430,8 +499,9 @@ async function crawlAndGenerate(options, onProgress) {
 				files: processedFiles,
 				siteName,
 				description,
-				origin: origin || firstUrl.origin,
-				generateFull: generateLlmsFullTxt
+				origin: origin$1 || firstUrl.origin,
+				generateFull: generateLlmsFullTxt,
+				outputDir
 			});
 			if (generateLlmsTxt) {
 				progress.generation.current = "Writing llms.txt";

package/dist/cli.mjs

@@ -1,9 +1,9 @@
-import { crawlAndGenerate, parseUrlPattern, validateGlobPattern, withHttps } from "./_chunks/crawl-B5MaCj6O.mjs";
+import { crawlAndGenerate, parseUrlPattern, validateGlobPattern, withHttps } from "./_chunks/crawl-BwURA9nQ.mjs";
 import { readFileSync } from "node:fs";
-import { dirname, join, resolve } from "pathe";
-import { fileURLToPath } from "node:url";
 import * as p$1 from "@clack/prompts";
 import * as p from "@clack/prompts";
+import { dirname, join, resolve } from "pathe";
+import { fileURLToPath } from "node:url";
 import { addDependency } from "nypm";
 
 //#region src/playwright-utils.ts
@@ -59,7 +59,7 @@ const packageJson = JSON.parse(readFileSync(packageJsonPath, "utf-8"));
 const version = packageJson.version;
 async function interactiveCrawl() {
 	console.clear();
-	p.intro("☁️  @mdream/crawl");
+	p.intro(`☁️  @mdream/crawl v${version}`);
 	const urlsInput = await p.text({
 		message: "Enter starting URL for crawling (supports glob patterns):",
 		placeholder: "e.g. docs.example.com, site.com/docs/**",
@@ -94,7 +94,7 @@ async function interactiveCrawl() {
 		p.cancel(error instanceof Error ? error.message : "Invalid URL pattern");
 		return null;
 	}
-	const outputDir = ".";
+	const outputDir = "output";
 	const crawlerOptions = await p.group({
 		driver: () => p.select({
 			message: "Select crawler driver:",
@@ -172,7 +172,8 @@ async function interactiveCrawl() {
 		`Follow links: Yes (depth ${crawlerOptions.maxDepth})`,
 		`Output formats: ${outputFormats.join(", ")}`,
 		`Sitemap discovery: Automatic`,
-		inferredOrigin && `Origin: ${inferredOrigin}`
+		inferredOrigin && `Origin: ${inferredOrigin}`,
+		advancedOptions.verbose && `Verbose logging: Enabled`
 	].filter(Boolean);
 	p.note(summary.join("\n"), "Crawl Configuration");
 	const shouldProceed = await p.confirm({
@@ -194,19 +195,18 @@ async function interactiveCrawl() {
 		generateLlmsFullTxt: advancedOptions.outputFormats.includes("llms-full.txt"),
 		generateIndividualMd: advancedOptions.outputFormats.includes("markdown"),
 		origin: inferredOrigin,
-		globPatterns
+		globPatterns,
+		verbose: advancedOptions.verbose
 	};
 }
-async function showCrawlResults(successful, failed, outputDir, generatedFiles, durationMinutes, remainingSeconds) {
+async function showCrawlResults(successful, failed, outputDir, generatedFiles, durationSeconds) {
 	const messages = [];
-	if (successful > 0) {
-		const durationStr = durationMinutes > 0 ? `${durationMinutes}m ${remainingSeconds}s` : `${remainingSeconds}s`;
-		messages.push(`✅ ${successful} pages processed in ${durationStr}`);
-	}
-	if (failed > 0) messages.push(`❌ ${failed} pages failed`);
-	if (generatedFiles.length > 0) messages.push(`📄 Generated: ${generatedFiles.join(", ")}`);
-	messages.push(`📁 Output: ${outputDir}`);
-	p.note(messages.join("\n"), "Completed: Results");
+	const durationStr = `${durationSeconds.toFixed(1)}s`;
+	const stats = failed > 0 ? `${successful} pages, ${failed} failed` : `${successful} pages`;
+	messages.push(`📄 ${stats} • ⏱️  ${durationStr}`);
+	messages.push(`📦 ${generatedFiles.join(", ")}`);
+	messages.push(`📁 ${outputDir}`);
+	p.note(messages.join("\n"), "✅ Complete");
 }
 function parseCliArgs() {
 	const args = process.argv.slice(2);
@@ -222,7 +222,7 @@ Usage:
 
 Options:
   -u, --url <url>              Website URL to crawl
-  -o, --output <dir>           Output directory (default: .)
+  -o, --output <dir>           Output directory (default: output)
   -d, --depth <number>         Crawl depth (default: 3)
   --driver <http|playwright>   Crawler driver (default: http)
   --artifacts <list>           Comma-separated list of artifacts: llms.txt,llms-full.txt,markdown (default: all)
@@ -232,6 +232,7 @@ Options:
   --max-pages <number>        Maximum pages to crawl (default: unlimited)
   --crawl-delay <seconds>     Crawl delay in seconds
   --exclude <pattern>         Exclude URLs matching glob patterns (can be used multiple times)
+  -v, --verbose               Enable verbose logging
   -h, --help                  Show this help message
   --version                   Show version number
 
@@ -241,6 +242,7 @@ Examples:
   @mdream/crawl -u harlanzw.com --artifacts "llms.txt,markdown"
   @mdream/crawl --url https://docs.example.com --depth 2 --artifacts "llms-full.txt"
   @mdream/crawl -u example.com --exclude "*/admin/*" --exclude "*/api/*"
+  @mdream/crawl -u example.com --verbose
 `);
 		process.exit(0);
 	}
@@ -349,9 +351,10 @@ Examples:
 	const siteNameOverride = getArgValue("--site-name");
 	const descriptionOverride = getArgValue("--description");
 	const patterns = [parsed];
+	const verbose = args.includes("--verbose") || args.includes("-v");
 	return {
 		urls: [url],
-		outputDir: resolve(getArgValue("--output") || getArgValue("-o") || "."),
+		outputDir: resolve(getArgValue("--output") || getArgValue("-o") || "output"),
 		driver: driver || "http",
 		maxRequestsPerCrawl: Number.parseInt(maxPagesStr || String(Number.MAX_SAFE_INTEGER)),
 		followLinks: true,
@@ -364,7 +367,8 @@ Examples:
 		origin: inferredOrigin,
 		globPatterns: patterns,
 		crawlDelay: crawlDelayStr ? Number.parseInt(crawlDelayStr) : void 0,
-		exclude: excludePatterns.length > 0 ? excludePatterns : void 0
+		exclude: excludePatterns.length > 0 ? excludePatterns : void 0,
+		verbose
 	};
 }
 async function main() {
@@ -383,7 +387,8 @@ async function main() {
 			`Driver: ${options.driver}`,
 			`Depth: ${options.maxDepth}`,
 			`Formats: ${formats.join(", ")}`,
-			options.exclude && options.exclude.length > 0 && `Exclude: ${options.exclude.join(", ")}`
+			options.exclude && options.exclude.length > 0 && `Exclude: ${options.exclude.join(", ")}`,
+			options.verbose && `Verbose: Enabled`
 		].filter(Boolean);
 		p.note(summary.join("\n"), "Configuration");
 	} else options = await interactiveCrawl();
@@ -419,9 +424,7 @@ async function main() {
 	s.stop();
 	const endTime = Date.now();
 	const durationMs = endTime - startTime;
-	const durationSeconds = Math.floor(durationMs / 1e3);
-	const durationMinutes = Math.floor(durationSeconds / 60);
-	const remainingSeconds = durationSeconds % 60;
+	const durationSeconds = durationMs / 1e3;
 	const successful = results.filter((r) => r.success).length;
 	const failed = results.filter((r) => !r.success).length;
 	const failedResults = results.filter((r) => !r.success);
@@ -442,17 +445,9 @@ async function main() {
 		if (options.generateLlmsFullTxt) generatedFiles.push("llms-full.txt");
 		if (options.generateIndividualMd) generatedFiles.push(`${successful} MD files`);
 	}
-	if (!cliOptions) await showCrawlResults(successful, failed, options.outputDir, generatedFiles, durationMinutes, remainingSeconds);
+	if (!cliOptions) await showCrawlResults(successful, failed, options.outputDir, generatedFiles, durationSeconds);
 	else {
-		const messages = [];
-		if (successful > 0) {
-			const durationStr = durationMinutes > 0 ? `${durationMinutes}m ${remainingSeconds}s` : `${remainingSeconds}s`;
-			messages.push(`✅ ${successful} pages processed in ${durationStr}`);
-		}
-		if (failed > 0) messages.push(`❌ ${failed} pages failed`);
-		if (generatedFiles.length > 0) messages.push(`📄 Generated: ${generatedFiles.join(", ")}`);
-		messages.push(`📁 Output: ${options.outputDir}`);
-		p.note(messages.join("\n"), "Completed: Results");
+		await showCrawlResults(successful, failed, options.outputDir, generatedFiles, durationSeconds);
 		if (successful === 0) process.exit(1);
 	}
 }

package/dist/index.d.mts

@@ -16,6 +16,7 @@ interface CrawlOptions {
   exclude?: string[];
   siteNameOverride?: string;
   descriptionOverride?: string;
+  verbose?: boolean;
 }
 interface ParsedUrlPattern {
   baseUrl: string;

package/dist/index.mjs

@@ -1,4 +1,4 @@
-import { crawlAndGenerate } from "./_chunks/crawl-B5MaCj6O.mjs";
+import { crawlAndGenerate } from "./_chunks/crawl-BwURA9nQ.mjs";
 import { writeFile } from "node:fs/promises";
 import { basename, sep } from "pathe";
 

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@mdream/crawl",
   "type": "module",
-  "version": "0.7.1",
+  "version": "0.8.0",
   "description": "Mdream Crawl generates comprehensive llms.txt artifacts from a single URL, using mdream to convert HTML to Markdown.",
   "author": {
     "name": "Harlan Wilton",
@@ -46,11 +46,11 @@
   },
   "dependencies": {
     "@clack/prompts": "^0.11.0",
-    "crawlee": "^3.13.9",
+    "crawlee": "^3.13.10",
     "nypm": "^0.6.0",
     "pathe": "^2.0.3",
     "picomatch": "^4.0.2",
-    "mdream": "0.7.1"
+    "mdream": "0.8.0"
   },
   "devDependencies": {
     "@types/picomatch": "^4.0.0"