@mdream/crawl 0.7.0

package/LICENSE.md

@@ -0,0 +1,9 @@
+MIT License
+
+Copyright (c) 2025 Harlan Wilton
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

package/README.md

@@ -0,0 +1,102 @@
+# @mdream/crawl
+
+Multi-page website crawler that generates comprehensive llms.txt files by following internal links and processing entire websites using mdream HTML-to-Markdown conversion.
+
+> **Note**: For single-page HTML-to-Markdown conversion, use the [`mdream`](../mdream) binary instead. `@mdream/crawl` is specifically designed for crawling entire websites with multiple pages.
+
+## Installation
+
+```bash
+npm install @mdream/crawl
+```
+
+## Usage
+
+Simply run the command to start the interactive multi-page website crawler:
+
+```bash
+npx @mdream/crawl
+```
+
+The crawler will automatically discover and follow internal links to crawl entire websites. The interactive interface provides:
+- ✨ Beautiful prompts powered by Clack
+- 🎯 Step-by-step configuration guidance
+- ✅ Input validation and helpful hints
+- 📋 Configuration summary before crawling
+- 🎉 Clean result display with progress indicators
+- 🧹 Automatic cleanup of crawler storage
+
+## Programmatic Usage
+
+You can also use @mdream/crawl programmatically in your Node.js applications:
+
+```typescript
+import { crawlAndGenerate, generateLlmsTxt } from '@mdream/crawl'
+
+// Crawl entire websites programmatically
+const results = await crawlAndGenerate({
+  urls: ['https://docs.example.com'], // Starting URLs for website crawling
+  outputDir: './output',
+  maxRequestsPerCrawl: 100, // Maximum pages per website
+  generateLlmsTxt: true,
+  followLinks: true, // Follow internal links to crawl entire site
+  maxDepth: 3, // How deep to follow links
+  driver: 'http', // or 'playwright' for JS-heavy sites
+  verbose: true
+})
+
+// Generate llms.txt manually from existing results
+await generateLlmsTxt({
+  siteName: 'Example Site',
+  description: 'Documentation for Example Site',
+  results: crawlResults,
+  outputPath: './output/llms.txt'
+})
+```
+
+## Output
+
+The crawler generates comprehensive output from entire websites:
+
+1. **Markdown files** - One `.md` file per crawled page with clean markdown content
+2. **llms.txt** - Comprehensive site overview file following the [llms.txt specification](https://llmstxt.org/)
+
+### Example llms.txt output
+
+```markdown
+# example.com
+
+## Pages
+
+- [Example Domain](https---example-com-.md): https://example.com/
+- [About Us](https---example-com-about.md): https://example.com/about
+```
+
+## Features
+
+- ✅ **Multi-Page Website Crawling**: Designed specifically for crawling entire websites by following internal links
+- ✅ **Purely Interactive**: No complex command-line options to remember
+- ✅ **Dual Crawler Support**: Fast HTTP crawler (default) + Playwright for JavaScript-heavy sites
+- ✅ **Smart Link Discovery**: Uses mdream's extraction plugin to find and follow internal links
+- ✅ **Rich Metadata Extraction**: Extracts titles, descriptions, keywords, and author info from all pages
+- ✅ **Comprehensive llms.txt Generation**: Creates complete site documentation files
+- ✅ **Configurable Depth Crawling**: Follow links with customizable depth limits (1-10 levels)
+- ✅ **Clean Markdown Conversion**: Powered by mdream's HTML-to-Markdown engine
+- ✅ **Performance Optimized**: HTTP crawler is 5-10x faster than browser-based crawling
+- ✅ **Beautiful Output**: Clean result display with progress indicators
+- ✅ **Automatic Cleanup**: Purges crawler storage after completion
+- ✅ **TypeScript Support**: Full type definitions with excellent IDE support
+
+## Use Cases
+
+Perfect for:
+- 📚 **Documentation Sites**: Crawl entire documentation websites (GitBook, Docusaurus, etc.)
+- 🏢 **Company Websites**: Generate comprehensive site overviews for LLM context
+- 📝 **Blogs**: Process entire blog archives with proper categorization
+- 🔗 **Multi-Page Resources**: Any website where you need all pages, not just one
+
+**Not suitable for**: Single-page conversions (use `mdream` binary instead)
+
+## License
+
+MIT

package/bin/mdream-crawl.mjs

@@ -0,0 +1,3 @@
+#!/usr/bin/env node
+
+import('../dist/cli.mjs')

package/dist/_chunks/crawl-NJU1Dyc-.mjs

@@ -0,0 +1,445 @@
+import { existsSync, mkdirSync } from "node:fs";
+import { writeFile } from "node:fs/promises";
+import { HttpCrawler, Sitemap, purgeDefaultStorages } from "crawlee";
+import { generateLlmsTxtArtifacts, htmlToMarkdown } from "mdream";
+import { withMinimalPreset } from "mdream/preset/minimal";
+import { dirname, join, normalize, resolve } from "pathe";
+import { minimatch } from "minimatch";
+import { extractionPlugin } from "mdream/plugins";
+
+//#region ../../node_modules/.pnpm/ufo@1.6.1/node_modules/ufo/dist/index.mjs
+const r = String.fromCharCode;
+const PROTOCOL_REGEX = /^[\s\w\0+.-]{2,}:([/\\]{2})?/;
+function withHttps(input) {
+	return withProtocol(input, "https://");
+}
+function withProtocol(input, protocol) {
+	let match = input.match(PROTOCOL_REGEX);
+	if (!match) match = input.match(/^\/{2,}/);
+	if (!match) return protocol + input;
+	return protocol + input.slice(match[0].length);
+}
+const protocolRelative = Symbol.for("ufo:protocolRelative");
+
+//#endregion
+//#region src/glob-utils.ts
+/**
+* Parse a URL that may contain glob patterns
+* Example: https://nuxtseo.com/docs/** -> { baseUrl: "https://nuxtseo.com", pattern: "/docs/**", isGlob: true }
+*/
+function parseUrlPattern(input) {
+	const hasGlob = input.includes("*") || input.includes("?") || input.includes("[");
+	if (!hasGlob) return {
+		baseUrl: input,
+		pattern: "",
+		isGlob: false
+	};
+	try {
+		const urlWithProtocol = input.startsWith("http") ? input : `https://${input}`;
+		const urlWithoutGlob = urlWithProtocol.replace(/\*.*$/, "");
+		const url = new URL(urlWithoutGlob);
+		const baseUrl = `${url.protocol}//${url.host}`;
+		const patternStart = input.indexOf(url.host) + url.host.length;
+		const pattern = input.substring(patternStart);
+		return {
+			baseUrl,
+			pattern,
+			isGlob: true
+		};
+	} catch (error) {
+		throw new Error(`Invalid URL pattern: "${input}". Please provide a valid URL with glob patterns (e.g., "example.com/docs/*" or "https://example.com/api/**").`);
+	}
+}
+/**
+* Check if a URL matches a glob pattern
+*/
+function matchesGlobPattern(url, parsedPattern) {
+	if (!parsedPattern.isGlob) return true;
+	try {
+		const urlObj = new URL(url);
+		const urlPath = urlObj.pathname + urlObj.search + urlObj.hash;
+		const urlBase = `${urlObj.protocol}//${urlObj.host}`;
+		if (urlBase !== parsedPattern.baseUrl) return false;
+		return minimatch(urlPath, parsedPattern.pattern);
+	} catch {
+		return false;
+	}
+}
+/**
+* Get the starting URL for crawling from a glob pattern
+* For https://nuxtseo.com/docs/**, we want to start at https://nuxtseo.com
+*/
+function getStartingUrl(parsedPattern) {
+	if (!parsedPattern.isGlob) return withHttps(parsedPattern.baseUrl);
+	const pattern = parsedPattern.pattern;
+	const firstGlobIndex = pattern.search(/[*?[]/);
+	if (firstGlobIndex === -1) return withHttps(parsedPattern.baseUrl + pattern);
+	const beforeGlob = pattern.substring(0, firstGlobIndex);
+	const lastSlash = beforeGlob.lastIndexOf("/");
+	const pathBeforeGlob = lastSlash >= 0 ? beforeGlob.substring(0, lastSlash + 1) : "/";
+	return withHttps(parsedPattern.baseUrl + pathBeforeGlob);
+}
+/**
+* Check if a URL should be excluded based on exclude patterns
+*/
+function isUrlExcluded(url, excludePatterns) {
+	if (!excludePatterns || excludePatterns.length === 0) return false;
+	try {
+		const urlObj = new URL(url);
+		const urlPath = urlObj.pathname + urlObj.search + urlObj.hash;
+		return excludePatterns.some((pattern) => {
+			if (pattern.includes("://")) {
+				const parsedPattern = parseUrlPattern(pattern);
+				if (parsedPattern.isGlob) return matchesGlobPattern(url, parsedPattern);
+				return url === pattern;
+			}
+			if (pattern.startsWith("/")) {
+				const adjustedPattern = pattern.endsWith("/*") ? pattern.replace("/*", "/**") : pattern;
+				return minimatch(urlPath, adjustedPattern);
+			}
+			return minimatch(urlPath, pattern) || minimatch(urlPath.substring(1), pattern);
+		});
+	} catch {
+		return false;
+	}
+}
+/**
+* Validate glob pattern syntax
+*/
+function validateGlobPattern(pattern) {
+	try {
+		parseUrlPattern(pattern);
+		return void 0;
+	} catch (error) {
+		return `Invalid glob pattern: ${error instanceof Error ? error.message : error}`;
+	}
+}
+
+//#endregion
+//#region src/metadata-extractor.ts
+function extractMetadata(html, url) {
+	const links = [];
+	let title = "";
+	let description = "";
+	let keywords = "";
+	let author = "";
+	const extractionPluginInstance = extractionPlugin({
+		"a[href]": (element) => {
+			const href = element.attributes?.href;
+			if (href) try {
+				const absoluteUrl = new URL(href, url).href;
+				if (!links.includes(absoluteUrl)) links.push(absoluteUrl);
+			} catch {}
+		},
+		"title": (element) => {
+			if (!title && element.textContent) title = element.textContent.trim();
+		},
+		"meta[name=\"description\"]": (element) => {
+			if (!description && element.attributes?.content) description = element.attributes.content.trim();
+		},
+		"meta[property=\"og:description\"]": (element) => {
+			if (!description && element.attributes?.content) description = element.attributes.content.trim();
+		},
+		"meta[name=\"keywords\"]": (element) => {
+			if (!keywords && element.attributes?.content) keywords = element.attributes.content.trim();
+		},
+		"meta[name=\"author\"]": (element) => {
+			if (!author && element.attributes?.content) author = element.attributes.content.trim();
+		},
+		"meta[property=\"og:title\"]": (element) => {
+			if (!title && element.attributes?.content) title = element.attributes.content.trim();
+		}
+	});
+	htmlToMarkdown(html, {
+		plugins: [extractionPluginInstance],
+		origin: new URL(url).origin
+	});
+	return {
+		title: title || new URL(url).pathname,
+		description: description || void 0,
+		keywords: keywords || void 0,
+		author: author || void 0,
+		links: links.filter((link) => {
+			try {
+				const linkUrl = new URL(link);
+				const baseUrl = new URL(url);
+				return linkUrl.hostname === baseUrl.hostname;
+			} catch {
+				return false;
+			}
+		})
+	};
+}
+
+//#endregion
+//#region src/crawl.ts
+async function crawlAndGenerate(options, onProgress) {
+	const { urls, outputDir: rawOutputDir, maxRequestsPerCrawl = Number.MAX_SAFE_INTEGER, generateLlmsTxt = true, generateLlmsFullTxt = false, generateIndividualMd = true, origin, driver = "http", followLinks = false, maxDepth = 1, globPatterns = [], crawlDelay, exclude = [], siteNameOverride, descriptionOverride } = options;
+	const outputDir = resolve(normalize(rawOutputDir));
+	let patterns;
+	try {
+		patterns = globPatterns.length > 0 ? globPatterns : urls.map(parseUrlPattern);
+	} catch (error) {
+		throw new Error(`Invalid URL pattern: ${error instanceof Error ? error.message : "Unknown error"}`);
+	}
+	let startingUrls = patterns.map(getStartingUrl);
+	const progress = {
+		sitemap: {
+			status: "discovering",
+			found: 0,
+			processed: 0
+		},
+		crawling: {
+			status: "starting",
+			total: 0,
+			processed: 0
+		},
+		generation: { status: "idle" }
+	};
+	if (startingUrls.length > 0) {
+		const baseUrl = new URL(startingUrls[0]).origin;
+		const homePageUrl = baseUrl;
+		onProgress?.(progress);
+		const robotsUrl = new URL("/robots.txt", baseUrl).toString();
+		const robotsResponse = await fetch(robotsUrl);
+		if (robotsResponse.ok) {
+			const robotsContent = await robotsResponse.text();
+			const sitemapMatches = robotsContent.match(/Sitemap:\s*(.*)/gi);
+			if (sitemapMatches && sitemapMatches.length > 0) {
+				progress.sitemap.found = sitemapMatches.length;
+				progress.sitemap.status = "processing";
+				onProgress?.(progress);
+				const robotsSitemaps = sitemapMatches.map((match) => match.replace(/Sitemap:\s*/i, "").trim());
+				for (const sitemapUrl of robotsSitemaps) try {
+					const { urls: robotsUrls } = await Sitemap.load(sitemapUrl);
+					const hasGlobPatterns = patterns.some((p) => p.isGlob);
+					if (hasGlobPatterns) {
+						const filteredUrls = robotsUrls.filter((url) => {
+							return !isUrlExcluded(url, exclude) && patterns.some((pattern) => matchesGlobPattern(url, pattern));
+						});
+						startingUrls = filteredUrls;
+						progress.sitemap.processed = filteredUrls.length;
+						onProgress?.(progress);
+						break;
+					} else {
+						const filteredUrls = robotsUrls.filter((url) => {
+							return !isUrlExcluded(url, exclude);
+						});
+						if (filteredUrls.length > 0) {
+							startingUrls = filteredUrls;
+							progress.sitemap.processed = filteredUrls.length;
+							onProgress?.(progress);
+							break;
+						}
+					}
+				} catch {
+					continue;
+				}
+			}
+		}
+		try {
+			const { urls: sitemapUrls } = await Sitemap.load(`${baseUrl}/sitemap.xml`);
+			const hasGlobPatterns = patterns.some((p) => p.isGlob);
+			if (hasGlobPatterns) {
+				const filteredUrls = sitemapUrls.filter((url) => {
+					return !isUrlExcluded(url, exclude) && patterns.some((pattern) => matchesGlobPattern(url, pattern));
+				});
+				startingUrls = filteredUrls;
+				progress.sitemap.found = sitemapUrls.length;
+				progress.sitemap.processed = filteredUrls.length;
+				onProgress?.(progress);
+			} else {
+				const filteredUrls = sitemapUrls.filter((url) => {
+					return !isUrlExcluded(url, exclude);
+				});
+				if (filteredUrls.length > 0) {
+					startingUrls = filteredUrls;
+					progress.sitemap.found = sitemapUrls.length;
+					progress.sitemap.processed = filteredUrls.length;
+					onProgress?.(progress);
+				}
+			}
+		} catch {
+			const commonSitemaps = [
+				`${baseUrl}/sitemap_index.xml`,
+				`${baseUrl}/sitemaps.xml`,
+				`${baseUrl}/sitemap-index.xml`
+			];
+			for (const sitemapUrl of commonSitemaps) try {
+				const { urls: altUrls } = await Sitemap.load(sitemapUrl);
+				const hasGlobPatterns = patterns.some((p) => p.isGlob);
+				if (hasGlobPatterns) {
+					const filteredUrls = altUrls.filter((url) => {
+						return !isUrlExcluded(url, exclude) && patterns.some((pattern) => matchesGlobPattern(url, pattern));
+					});
+					startingUrls = filteredUrls;
+					progress.sitemap.found = altUrls.length;
+					progress.sitemap.processed = filteredUrls.length;
+					onProgress?.(progress);
+					break;
+				} else {
+					const filteredUrls = altUrls.filter((url) => {
+						return !isUrlExcluded(url, exclude);
+					});
+					if (filteredUrls.length > 0) {
+						startingUrls = filteredUrls;
+						progress.sitemap.found = altUrls.length;
+						progress.sitemap.processed = filteredUrls.length;
+						onProgress?.(progress);
+						break;
+					}
+				}
+			} catch {
+				continue;
+			}
+		}
+		if (!startingUrls.includes(homePageUrl)) startingUrls.unshift(homePageUrl);
+		progress.sitemap.status = "completed";
+		progress.crawling.total = startingUrls.length;
+		onProgress?.(progress);
+	}
+	if (!existsSync(outputDir)) mkdirSync(outputDir, { recursive: true });
+	const results = [];
+	const processedUrls = new Set();
+	const shouldCrawlUrl = (url) => {
+		if (isUrlExcluded(url, exclude)) return false;
+		if (!patterns.some((p) => p.isGlob)) return true;
+		return patterns.some((pattern) => matchesGlobPattern(url, pattern));
+	};
+	const createRequestHandler = (crawlerType) => {
+		return async ({ request, body, page, enqueueLinks }) => {
+			const startTime = Date.now();
+			progress.crawling.currentUrl = request.loadedUrl;
+			onProgress?.(progress);
+			const baseUrl = new URL(startingUrls[0]).origin;
+			const homePageUrl = baseUrl;
+			let html;
+			let title;
+			if (crawlerType === "playwright") {
+				await page.waitForLoadState("networkidle");
+				title = await page.title();
+				html = await page.innerHTML("html");
+			} else {
+				html = typeof body === "string" ? body : body.toString();
+				title = "";
+			}
+			const metadata = extractMetadata(html, request.loadedUrl);
+			if (!title) title = metadata.title;
+			const shouldProcessMarkdown = shouldCrawlUrl(request.loadedUrl);
+			let md = "";
+			if (shouldProcessMarkdown) md = htmlToMarkdown(html, withMinimalPreset({ origin: origin || new URL(request.loadedUrl).origin }));
+			let filePath;
+			if (shouldProcessMarkdown) {
+				const urlObj = new URL(request.loadedUrl);
+				const urlPath = urlObj.pathname === "/" ? "/index" : urlObj.pathname;
+				const pathSegments = urlPath.replace(/\/$/, "").split("/").filter((seg) => seg.length > 0);
+				const safeSegments = pathSegments.map((seg) => seg.replace(/[^\w\-]/g, "-"));
+				const filename = safeSegments.length > 0 ? safeSegments.join("/") : "index";
+				const safeFilename = normalize(`${filename}.md`);
+				filePath = join(outputDir, "md", safeFilename);
+				if (generateIndividualMd) {
+					const fileDir = dirname(filePath);
+					if (fileDir && !existsSync(fileDir)) mkdirSync(fileDir, { recursive: true });
+					await writeFile(filePath, md, "utf-8");
+				}
+			}
+			const isHomePage = request.loadedUrl === homePageUrl;
+			if (shouldProcessMarkdown || isHomePage) {
+				const result = {
+					url: request.loadedUrl,
+					title,
+					content: md,
+					filePath: generateIndividualMd && shouldProcessMarkdown ? filePath : void 0,
+					timestamp: startTime,
+					success: true,
+					metadata,
+					depth: request.userData?.depth || 0
+				};
+				results.push(result);
+				progress.crawling.processed = results.length;
+				onProgress?.(progress);
+			}
+			if (followLinks && (request.userData?.depth || 0) < maxDepth) {
+				const currentDepth = (request.userData?.depth || 0) + 1;
+				const filteredLinks = metadata.links.filter((link) => {
+					return shouldCrawlUrl(link);
+				});
+				if (enqueueLinks) await enqueueLinks({
+					urls: filteredLinks,
+					userData: { depth: currentDepth }
+				});
+				else for (const link of filteredLinks) if (!processedUrls.has(link)) processedUrls.add(link);
+			}
+		};
+	};
+	let crawler;
+	const crawlerOptions = {
+		requestHandler: createRequestHandler(driver),
+		maxRequestsPerCrawl,
+		respectRobotsTxtFile: true
+	};
+	if (crawlDelay) crawlerOptions.requestHandlerTimeoutMillis = crawlDelay * 1e3;
+	if (driver === "playwright") {
+		const { PlaywrightCrawler: PlaywrightCrawlerClass } = await import("crawlee");
+		crawler = new PlaywrightCrawlerClass(crawlerOptions);
+	} else crawler = new HttpCrawler(crawlerOptions);
+	const initialRequests = startingUrls.map((url) => ({
+		url,
+		userData: { depth: 0 }
+	}));
+	progress.crawling.status = "processing";
+	progress.crawling.total = startingUrls.length;
+	onProgress?.(progress);
+	await crawler.run(initialRequests);
+	progress.crawling.status = "completed";
+	onProgress?.(progress);
+	if (results.some((r$1) => r$1.success)) {
+		progress.generation.status = "generating";
+		onProgress?.(progress);
+		const successfulResults = results.filter((r$1) => r$1.success);
+		const firstUrl = new URL(withHttps(urls[0]));
+		const homePageResult = successfulResults.find((r$1) => {
+			const resultUrl = new URL(withHttps(r$1.url));
+			const homeUrl = new URL(withHttps(urls[0]));
+			return resultUrl.href === homeUrl.href;
+		});
+		const siteName = siteNameOverride || homePageResult?.metadata?.title || firstUrl.hostname;
+		const description = descriptionOverride || homePageResult?.metadata?.description || successfulResults[0]?.metadata?.description;
+		if (generateLlmsTxt || generateLlmsFullTxt) {
+			progress.generation.current = "Generating llms.txt files";
+			onProgress?.(progress);
+			const contentResults = successfulResults.filter((result) => result.content && result.content.trim().length > 0);
+			const processedFiles = contentResults.map((result) => ({
+				filePath: result.filePath,
+				title: result.title,
+				content: result.content,
+				url: result.url,
+				metadata: result.metadata
+			}));
+			const llmsResult = await generateLlmsTxtArtifacts({
+				files: processedFiles,
+				siteName,
+				description,
+				origin: origin || firstUrl.origin,
+				generateFull: generateLlmsFullTxt
+			});
+			if (generateLlmsTxt) {
+				progress.generation.current = "Writing llms.txt";
+				onProgress?.(progress);
+				await writeFile(join(outputDir, "llms.txt"), llmsResult.llmsTxt, "utf-8");
+			}
+			if (generateLlmsFullTxt && llmsResult.llmsFullTxt) {
+				progress.generation.current = "Writing llms-full.txt";
+				onProgress?.(progress);
+				await writeFile(join(outputDir, "llms-full.txt"), llmsResult.llmsFullTxt, "utf-8");
+			}
+		}
+		progress.generation.status = "completed";
+		onProgress?.(progress);
+	}
+	await purgeDefaultStorages();
+	return results;
+}
+
+//#endregion
+export { crawlAndGenerate, parseUrlPattern, validateGlobPattern, withHttps };

package/dist/cli.d.mts

@@ -0,0 +1 @@
+export { };

package/dist/cli.mjs

@@ -0,0 +1,458 @@
+import { crawlAndGenerate, parseUrlPattern, validateGlobPattern, withHttps } from "./_chunks/crawl-NJU1Dyc-.mjs";
+import { readFileSync } from "node:fs";
+import { dirname, join, resolve } from "pathe";
+import { fileURLToPath } from "node:url";
+import * as p$1 from "@clack/prompts";
+import * as p from "@clack/prompts";
+import { addDependency } from "nypm";
+
+//#region src/playwright-utils.ts
+async function checkPlaywrightInstallation() {
+	try {
+		await import("playwright");
+		return true;
+	} catch {
+		return false;
+	}
+}
+async function promptPlaywrightInstall() {
+	const shouldInstall = await p$1.confirm({
+		message: "Playwright is required for the Playwright driver. Install it now?",
+		initialValue: true
+	});
+	if (p$1.isCancel(shouldInstall) || !shouldInstall) return false;
+	const s = p$1.spinner();
+	s.start("Installing Playwright...");
+	try {
+		await addDependency("playwright", { workspace: true });
+		s.stop("Playwright installed successfully!");
+		return true;
+	} catch {
+		try {
+			await addDependency("playwright");
+			s.stop("Playwright installed successfully!");
+			return true;
+		} catch (fallbackError) {
+			s.stop("Failed to install Playwright");
+			p$1.log.error(`Installation failed: ${fallbackError}`);
+			return false;
+		}
+	}
+}
+async function ensurePlaywrightInstalled() {
+	const isInstalled = await checkPlaywrightInstallation();
+	if (isInstalled) return true;
+	p$1.log.warn("Playwright driver selected but Playwright is not installed.");
+	const installed = await promptPlaywrightInstall();
+	if (!installed) {
+		p$1.log.error("Cannot proceed with Playwright driver without Playwright installed.");
+		return false;
+	}
+	return true;
+}
+
+//#endregion
+//#region src/cli.ts
+const __dirname = dirname(fileURLToPath(import.meta.url));
+const packageJsonPath = join(__dirname, "..", "package.json");
+const packageJson = JSON.parse(readFileSync(packageJsonPath, "utf-8"));
+const version = packageJson.version;
+async function interactiveCrawl() {
+	console.clear();
+	p.intro("☁️  @mdream/crawl");
+	const urlsInput = await p.text({
+		message: "Enter starting URL for crawling (supports glob patterns):",
+		placeholder: "e.g. docs.example.com, site.com/docs/**",
+		validate: (value) => {
+			if (!value) return "Please enter at least one URL";
+			const urls$1 = value.split(",").map((url) => url.trim());
+			for (const url of urls$1) {
+				const globError = validateGlobPattern(url);
+				if (globError) return globError;
+				try {
+					const parsed = parseUrlPattern(url);
+					if (!parsed.isGlob) try {
+						new URL(withHttps(url));
+					} catch {
+						return `Invalid URL: ${withHttps(url)}`;
+					}
+				} catch (error) {
+					return error instanceof Error ? error.message : "Invalid URL pattern";
+				}
+			}
+		}
+	});
+	if (p.isCancel(urlsInput)) {
+		p.cancel("Operation cancelled.");
+		return null;
+	}
+	const urls = urlsInput.split(",").map((url) => url.trim());
+	let globPatterns;
+	try {
+		globPatterns = urls.map(parseUrlPattern);
+	} catch (error) {
+		p.cancel(error instanceof Error ? error.message : "Invalid URL pattern");
+		return null;
+	}
+	const outputDir = ".";
+	const crawlerOptions = await p.group({
+		driver: () => p.select({
+			message: "Select crawler driver:",
+			options: [{
+				value: "http",
+				label: "HTTP Crawler (Fast, for static content)",
+				hint: "Recommended"
+			}, {
+				value: "playwright",
+				label: "Playwright (Slower, supports JavaScript)"
+			}],
+			initialValue: "http"
+		}),
+		maxDepth: () => p.text({
+			message: "Clicks to page (crawl depth):",
+			placeholder: "3",
+			defaultValue: "3",
+			validate: (value) => {
+				const num = Number.parseInt(value);
+				if (Number.isNaN(num) || num < 1 || num > 10) return "Depth must be between 1 and 10";
+			}
+		})
+	}, { onCancel: () => {
+		p.cancel("Operation cancelled.");
+		process.exit(0);
+	} });
+	const advancedOptions = await p.group({ outputFormats: () => p.multiselect({
+		message: "Select output formats:",
+		options: [
+			{
+				value: "llms.txt",
+				label: "llms.txt (basic format)",
+				hint: "Recommended"
+			},
+			{
+				value: "llms-full.txt",
+				label: "llms-full.txt (extended format)"
+			},
+			{
+				value: "markdown",
+				label: "Individual Markdown files"
+			}
+		],
+		initialValues: [
+			"llms.txt",
+			"llms-full.txt",
+			"markdown"
+		]
+	}) }, { onCancel: () => {
+		p.cancel("Operation cancelled.");
+		process.exit(0);
+	} });
+	const firstUrl = urls[0];
+	const inferredOrigin = (() => {
+		try {
+			const url = new URL(withHttps(firstUrl));
+			return `${url.protocol}//${url.host}`;
+		} catch {
+			return void 0;
+		}
+	})();
+	const outputFormats = advancedOptions.outputFormats.map((f) => {
+		switch (f) {
+			case "llms.txt": return "llms.txt";
+			case "llms-full.txt": return "llms-full.txt";
+			case "markdown": return "Individual MD files";
+			default: return f;
+		}
+	});
+	const summary = [
+		`URLs: ${urls.join(", ")}`,
+		`Output: ${outputDir}`,
+		`Driver: ${crawlerOptions.driver}`,
+		`Max pages: Unlimited`,
+		`Follow links: Yes (depth ${crawlerOptions.maxDepth})`,
+		`Output formats: ${outputFormats.join(", ")}`,
+		`Sitemap discovery: Automatic`,
+		inferredOrigin && `Origin: ${inferredOrigin}`
+	].filter(Boolean);
+	p.note(summary.join("\n"), "Crawl Configuration");
+	const shouldProceed = await p.confirm({
+		message: "Start crawling?",
+		initialValue: true
+	});
+	if (p.isCancel(shouldProceed) || !shouldProceed) {
+		p.cancel("Crawl cancelled.");
+		return null;
+	}
+	return {
+		urls,
+		outputDir: resolve(outputDir),
+		driver: crawlerOptions.driver,
+		maxRequestsPerCrawl: Number.MAX_SAFE_INTEGER,
+		followLinks: true,
+		maxDepth: Number.parseInt(crawlerOptions.maxDepth),
+		generateLlmsTxt: advancedOptions.outputFormats.includes("llms.txt"),
+		generateLlmsFullTxt: advancedOptions.outputFormats.includes("llms-full.txt"),
+		generateIndividualMd: advancedOptions.outputFormats.includes("markdown"),
+		origin: inferredOrigin,
+		globPatterns
+	};
+}
+async function showCrawlResults(successful, failed, outputDir, generatedFiles) {
+	const messages = [];
+	if (successful > 0) messages.push(`✅ ${successful} pages processed successfully`);
+	if (failed > 0) messages.push(`❌ ${failed} pages failed`);
+	if (generatedFiles.length > 0) messages.push(`📄 Generated: ${generatedFiles.join(", ")}`);
+	messages.push(`📁 Output: ${outputDir}`);
+	p.note(messages.join("\n"), "Crawl Results");
+	if (successful > 0) p.outro("🎉 Crawling completed successfully!");
+	else p.outro("❌ Crawling failed - no pages processed");
+}
+function parseCliArgs() {
+	const args = process.argv.slice(2);
+	if (args.includes("--help") || args.includes("-h")) {
+		console.log(`
+@mdream/crawl v${version}
+
+Multi-page website crawler that generates comprehensive llms.txt files
+
+Usage:
+  @mdream/crawl [options] <url>             Crawl a website with CLI flags
+  @mdream/crawl                             Start interactive mode
+
+Options:
+  -u, --url <url>              Website URL to crawl
+  -o, --output <dir>           Output directory (default: .)
+  -d, --depth <number>         Crawl depth (default: 3)
+  --driver <http|playwright>   Crawler driver (default: http)
+  --artifacts <list>           Comma-separated list of artifacts: llms.txt,llms-full.txt,markdown (default: all)
+  --origin <url>               Origin URL for resolving relative paths (overrides auto-detection)
+  --site-name <name>           Override site name (overrides auto-extracted title)
+  --description <desc>         Override site description (overrides auto-extracted description)
+  --max-pages <number>        Maximum pages to crawl (default: unlimited)
+  --crawl-delay <seconds>     Crawl delay in seconds
+  --exclude <pattern>         Exclude URLs matching glob patterns (can be used multiple times)
+  -h, --help                  Show this help message
+  --version                   Show version number
+
+Note: Sitemap discovery and robots.txt checking are automatic
+
+Examples:
+  @mdream/crawl -u harlanzw.com --artifacts "llms.txt,markdown"
+  @mdream/crawl --url https://docs.example.com --depth 2 --artifacts "llms-full.txt"
+  @mdream/crawl -u example.com --exclude "*/admin/*" --exclude "*/api/*"
+`);
+		process.exit(0);
+	}
+	if (args.includes("--version")) {
+		console.log(`@mdream/crawl v${version}`);
+		process.exit(0);
+	}
+	if (args.length === 0) return null;
+	const getArgValue = (flag) => {
+		const index = args.findIndex((arg) => arg === flag || arg === flag.replace("--", "-"));
+		return index >= 0 && index + 1 < args.length ? args[index + 1] : void 0;
+	};
+	const getArgValues = (flag) => {
+		const values = [];
+		for (let i = 0; i < args.length; i++) if (args[i] === flag || args[i] === flag.replace("--", "-")) {
+			if (i + 1 < args.length && !args[i + 1].startsWith("-")) values.push(args[i + 1]);
+		}
+		return values;
+	};
+	const urlFromFlag = getArgValue("--url") || getArgValue("-u");
+	const urlFromArgs = args.find((arg) => !arg.startsWith("-") && !args[args.indexOf(arg) - 1]?.startsWith("-"));
+	const url = urlFromFlag || urlFromArgs;
+	if (!url) {
+		p.log.error("Error: URL is required when using CLI arguments");
+		p.log.info("Use --help for usage information or run without arguments for interactive mode");
+		process.exit(1);
+	}
+	const globError = validateGlobPattern(url);
+	if (globError) {
+		p.log.error(`Error: ${globError}`);
+		process.exit(1);
+	}
+	let parsed;
+	try {
+		parsed = parseUrlPattern(url);
+	} catch (error) {
+		p.log.error(`Error: ${error instanceof Error ? error.message : "Invalid URL pattern"}`);
+		process.exit(1);
+	}
+	if (!parsed.isGlob) try {
+		new URL(withHttps(url));
+	} catch {
+		p.log.error(`Error: Invalid URL: ${withHttps(url)}`);
+		process.exit(1);
+	}
+	const excludePatterns = getArgValues("--exclude");
+	for (const pattern of excludePatterns) {
+		const excludeError = validateGlobPattern(pattern);
+		if (excludeError) {
+			p.log.error(`Error in exclude pattern: ${excludeError}`);
+			process.exit(1);
+		}
+	}
+	const depthStr = getArgValue("--depth") || getArgValue("-d") || "3";
+	const depth = Number.parseInt(depthStr);
+	if (Number.isNaN(depth) || depth < 1 || depth > 10) {
+		p.log.error("Error: Depth must be between 1 and 10");
+		process.exit(1);
+	}
+	const driver = getArgValue("--driver");
+	if (driver && driver !== "http" && driver !== "playwright") {
+		p.log.error("Error: Driver must be either \"http\" or \"playwright\"");
+		process.exit(1);
+	}
+	const maxPagesStr = getArgValue("--max-pages");
+	if (maxPagesStr) {
+		const maxPages = Number.parseInt(maxPagesStr);
+		if (Number.isNaN(maxPages) || maxPages < 1) {
+			p.log.error("Error: Max pages must be a positive number");
+			process.exit(1);
+		}
+	}
+	const crawlDelayStr = getArgValue("--crawl-delay");
+	if (crawlDelayStr) {
+		const crawlDelay = Number.parseInt(crawlDelayStr);
+		if (Number.isNaN(crawlDelay) || crawlDelay < 0) {
+			p.log.error("Error: Crawl delay must be a non-negative number");
+			process.exit(1);
+		}
+	}
+	const artifactsStr = getArgValue("--artifacts");
+	const artifacts = artifactsStr ? artifactsStr.split(",").map((a) => a.trim()) : [
+		"llms.txt",
+		"llms-full.txt",
+		"markdown"
+	];
+	const validArtifacts = [
+		"llms.txt",
+		"llms-full.txt",
+		"markdown"
+	];
+	for (const artifact of artifacts) if (!validArtifacts.includes(artifact)) {
+		p.log.error(`Error: Invalid artifact '${artifact}'. Valid options: ${validArtifacts.join(", ")}`);
+		process.exit(1);
+	}
+	const originOverride = getArgValue("--origin");
+	const inferredOrigin = (() => {
+		if (originOverride) return originOverride;
+		try {
+			const urlObj = new URL(withHttps(url));
+			return `${urlObj.protocol}//${urlObj.host}`;
+		} catch {
+			return void 0;
+		}
+	})();
+	const siteNameOverride = getArgValue("--site-name");
+	const descriptionOverride = getArgValue("--description");
+	const patterns = [parsed];
+	return {
+		urls: [url],
+		outputDir: resolve(getArgValue("--output") || getArgValue("-o") || "."),
+		driver: driver || "http",
+		maxRequestsPerCrawl: Number.parseInt(maxPagesStr || String(Number.MAX_SAFE_INTEGER)),
+		followLinks: true,
+		maxDepth: depth,
+		generateLlmsTxt: artifacts.includes("llms.txt"),
+		generateLlmsFullTxt: artifacts.includes("llms-full.txt"),
+		generateIndividualMd: artifacts.includes("markdown"),
+		siteNameOverride,
+		descriptionOverride,
+		origin: inferredOrigin,
+		globPatterns: patterns,
+		crawlDelay: crawlDelayStr ? Number.parseInt(crawlDelayStr) : void 0,
+		exclude: excludePatterns.length > 0 ? excludePatterns : void 0
+	};
+}
+async function main() {
+	const cliOptions = parseCliArgs();
+	let options;
+	if (cliOptions) {
+		options = cliOptions;
+		p.intro(`☁️  mdream v${version}`);
+		const formats = [];
+		if (options.generateLlmsTxt) formats.push("llms.txt");
+		if (options.generateLlmsFullTxt) formats.push("llms-full.txt");
+		if (options.generateIndividualMd) formats.push("Individual MD files");
+		const summary = [
+			`URL: ${options.urls.join(", ")}`,
+			`Output: ${options.outputDir}`,
+			`Driver: ${options.driver}`,
+			`Depth: ${options.maxDepth}`,
+			`Formats: ${formats.join(", ")}`,
+			options.exclude && options.exclude.length > 0 && `Exclude: ${options.exclude.join(", ")}`
+		].filter(Boolean);
+		p.note(summary.join("\n"), "Configuration");
+	} else options = await interactiveCrawl();
+	if (!options) process.exit(0);
+	if (options.driver === "playwright") {
+		const playwrightInstalled = await ensurePlaywrightInstalled();
+		if (!playwrightInstalled) {
+			p.log.error("Cannot proceed without Playwright. Please install it manually or use the HTTP driver instead.");
+			process.exit(1);
+		}
+	}
+	const s = p.spinner();
+	s.start("Starting crawl...");
+	const results = await crawlAndGenerate(options, (progress) => {
+		if (progress.sitemap.status === "discovering") s.message("Discovering sitemaps...");
+		else if (progress.sitemap.status === "processing") s.message(`Processing sitemap... Found ${progress.sitemap.found} URLs`);
+		else if (progress.crawling.status === "processing") {
+			const processedCount = progress.crawling.processed;
+			const totalCount = progress.crawling.total;
+			const currentUrl = progress.crawling.currentUrl;
+			if (currentUrl) {
+				const shortUrl = currentUrl.length > 60 ? `${currentUrl.substring(0, 57)}...` : currentUrl;
+				if (processedCount > totalCount) s.message(`Crawling ${processedCount}: ${shortUrl}`);
+				else s.message(`Crawling ${processedCount}/${totalCount}: ${shortUrl}`);
+			} else if (processedCount > totalCount) s.message(`Crawling... ${processedCount} pages`);
+			else s.message(`Crawling... ${processedCount}/${totalCount} pages`);
+		} else if (progress.generation.status === "generating") {
+			const current = progress.generation.current || "Generating files";
+			s.message(current);
+		}
+	});
+	s.stop("Crawl completed!");
+	const successful = results.filter((r) => r.success).length;
+	const failed = results.filter((r) => !r.success).length;
+	const failedResults = results.filter((r) => !r.success);
+	if (failed > 0 && cliOptions) {
+		p.log.error("Failed URLs:");
+		failedResults.forEach((result) => {
+			p.log.error(`  ${result.url}: ${result.error || "Unknown error"}`);
+		});
+	} else if (failed > 0) {
+		console.log("\nFailed URLs:");
+		failedResults.forEach((result) => {
+			console.log(`  - ${result.url}: ${result.error || "Unknown error"}`);
+		});
+	}
+	const generatedFiles = [];
+	if (successful > 0) {
+		if (options.generateLlmsTxt) generatedFiles.push("llms.txt");
+		if (options.generateLlmsFullTxt) generatedFiles.push("llms-full.txt");
+		if (options.generateIndividualMd) generatedFiles.push(`${successful} MD files`);
+	}
+	if (!cliOptions) await showCrawlResults(successful, failed, options.outputDir, generatedFiles);
+	else {
+		const messages = [];
+		if (successful > 0) messages.push(`✅ ${successful} pages processed`);
+		if (failed > 0) messages.push(`❌ ${failed} pages failed`);
+		if (generatedFiles.length > 0) messages.push(`📄 Generated: ${generatedFiles.join(", ")}`);
+		messages.push(`📁 Output: ${options.outputDir}`);
+		p.note(messages.join("\n"), "Results");
+		if (successful > 0) p.outro("🎉 Crawling completed!");
+		else {
+			p.outro("❌ Crawling failed - no pages processed");
+			process.exit(1);
+		}
+	}
+}
+main().catch((error) => {
+	p.log.error(`Unexpected error: ${error}`);
+	process.exit(1);
+});
+
+//#endregion

package/dist/index.d.mts

@@ -0,0 +1,74 @@
+//#region src/types.d.ts
+interface CrawlOptions {
+  urls: string[];
+  outputDir: string;
+  maxRequestsPerCrawl?: number;
+  generateLlmsTxt?: boolean;
+  generateLlmsFullTxt?: boolean;
+  generateIndividualMd?: boolean;
+  origin?: string;
+  chunkSize?: number;
+  driver?: 'http' | 'playwright';
+  followLinks?: boolean;
+  maxDepth?: number;
+  globPatterns?: ParsedUrlPattern[];
+  crawlDelay?: number;
+  exclude?: string[];
+  siteNameOverride?: string;
+  descriptionOverride?: string;
+}
+interface ParsedUrlPattern {
+  baseUrl: string;
+  pattern: string;
+  isGlob: boolean;
+}
+interface PageMetadata {
+  title: string;
+  description?: string;
+  keywords?: string;
+  author?: string;
+  links: string[];
+}
+interface CrawlResult {
+  url: string;
+  title: string;
+  content: string;
+  filePath?: string;
+  timestamp: number;
+  success: boolean;
+  error?: string;
+  metadata?: PageMetadata;
+  depth?: number;
+}
+interface LlmsTxtOptions {
+  siteName: string;
+  description?: string;
+  results: CrawlResult[];
+  outputPath: string;
+}
+//#endregion
+//#region src/crawl.d.ts
+interface CrawlProgress {
+  sitemap: {
+    status: 'discovering' | 'processing' | 'completed';
+    found: number;
+    processed: number;
+  };
+  crawling: {
+    status: 'starting' | 'processing' | 'completed';
+    total: number;
+    processed: number;
+    currentUrl?: string;
+  };
+  generation: {
+    status: 'idle' | 'generating' | 'completed';
+    current?: string;
+  };
+}
+declare function crawlAndGenerate(options: CrawlOptions, onProgress?: (progress: CrawlProgress) => void): Promise<CrawlResult[]>;
+//#endregion
+//#region src/llms-txt.d.ts
+declare function generateLlmsTxt(options: LlmsTxtOptions): Promise<void>;
+declare function generateLlmsFullTxt(options: LlmsTxtOptions): Promise<void>;
+//#endregion
+export { CrawlOptions, CrawlResult, LlmsTxtOptions, crawlAndGenerate, generateLlmsFullTxt, generateLlmsTxt };

package/dist/index.mjs

@@ -0,0 +1,66 @@
+import { crawlAndGenerate } from "./_chunks/crawl-NJU1Dyc-.mjs";
+import { writeFile } from "node:fs/promises";
+import { basename, sep } from "pathe";
+
+//#region src/llms-txt.ts
+async function generateLlmsTxt(options) {
+	const { siteName, description, results, outputPath } = options;
+	let content = `# ${siteName}\n\n`;
+	if (description) content += `> ${description}\n\n`;
+	if (results.length > 0) {
+		content += `## Pages\n\n`;
+		for (const result of results) {
+			let title;
+			try {
+				title = result.title || new URL(result.url).pathname;
+			} catch {
+				title = result.title || result.url;
+			}
+			if (result.filePath) {
+				const mdSeparator = `${sep}md${sep}`;
+				const mdIndex = result.filePath.indexOf(mdSeparator);
+				const relativePath = mdIndex !== -1 ? result.filePath.substring(mdIndex + mdSeparator.length) : basename(result.filePath);
+				const linkPath = relativePath.split(sep).join("/");
+				content += `- [${title}](md/${linkPath}): ${result.url}\n`;
+			} else {
+				const description$1 = result.metadata?.description ? result.metadata.description.split("\n")[0].substring(0, 100) + (result.metadata.description.length > 100 ? "..." : "") : "";
+				content += `- [${title}](${result.url})${description$1 ? `: ${description$1}` : ""}\n`;
+			}
+		}
+	}
+	await writeFile(outputPath, content, "utf-8");
+}
+async function generateLlmsFullTxt(options) {
+	const { siteName, description, results, outputPath } = options;
+	let content = `# ${siteName}\n\n`;
+	if (description) content += `> ${description}\n\n`;
+	if (results.length > 0) {
+		content += `## Table of Contents\n\n`;
+		for (const result of results) {
+			let title;
+			try {
+				title = result.title || new URL(result.url).pathname;
+			} catch {
+				title = result.title || result.url;
+			}
+			const anchor = title.toLowerCase().replace(/[^a-z0-9]/g, "-");
+			content += `- [${title}](#${anchor})\n`;
+		}
+		content += `\n---\n\n`;
+		for (const result of results) {
+			let title;
+			try {
+				title = result.title || new URL(result.url).pathname;
+			} catch {
+				title = result.title || result.url;
+			}
+			content += `## ${title}\n\n`;
+			content += `**URL:** ${result.url}\n\n`;
+			content += `${result.content}\n\n---\n\n`;
+		}
+	}
+	await writeFile(outputPath, content, "utf-8");
+}
+
+//#endregion
+export { crawlAndGenerate, generateLlmsFullTxt, generateLlmsTxt };

package/package.json

@@ -0,0 +1,62 @@
+{
+  "name": "@mdream/crawl",
+  "type": "module",
+  "version": "0.7.0",
+  "description": "Mdream Crawl generates comprehensive llms.txt artifacts from a single URL, using mdream to convert HTML to Markdown.",
+  "author": {
+    "name": "Harlan Wilton",
+    "email": "harlan@harlanzw.com",
+    "url": "https://harlanzw.com/"
+  },
+  "license": "MIT",
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.mts",
+      "import": {
+        "types": "./dist/index.d.mts",
+        "default": "./dist/index.mjs"
+      },
+      "default": "./dist/index.mjs"
+    },
+    "./cli": {
+      "types": "./dist/cli.d.mts",
+      "import": {
+        "types": "./dist/cli.d.mts",
+        "default": "./dist/cli.mjs"
+      },
+      "default": "./dist/cli.mjs"
+    }
+  },
+  "main": "./dist/index.mjs",
+  "types": "./dist/index.d.mts",
+  "bin": {
+    "@mdream/crawl": "./bin/mdream-crawl.mjs"
+  },
+  "files": [
+    "bin",
+    "dist"
+  ],
+  "peerDependencies": {
+    "playwright": "^1.53.2"
+  },
+  "peerDependenciesMeta": {
+    "playwright": {
+      "optional": true
+    }
+  },
+  "dependencies": {
+    "@clack/prompts": "^0.11.0",
+    "crawlee": "^3.13.9",
+    "minimatch": "^10.0.3",
+    "nypm": "^0.6.0",
+    "pathe": "^2.0.3",
+    "mdream": "0.7.0"
+  },
+  "scripts": {
+    "build": "obuild",
+    "typecheck": "tsc --noEmit",
+    "dev:prepare": "obuild --stub",
+    "test": "vitest test",
+    "test:attw": "attw --pack"
+  }
+}