@mdream/crawl 1.0.0-beta.9 → 1.0.1

package/dist/_chunks/playwright-utils.mjs

@@ -0,0 +1,59 @@
+import * as p from "@clack/prompts";
+import { addDependency } from "nypm";
+//#region src/playwright-utils.ts
+async function checkPlaywrightInstallation() {
+	try {
+		await import("playwright");
+		return true;
+	} catch {
+		return false;
+	}
+}
+async function promptPlaywrightInstall() {
+	const shouldInstall = await p.confirm({
+		message: "Playwright is required for the Playwright driver. Install it now?",
+		initialValue: true
+	});
+	if (p.isCancel(shouldInstall) || !shouldInstall) return false;
+	const s = p.spinner();
+	s.start("Installing Playwright globally...");
+	try {
+		await addDependency("playwright", { global: true });
+		s.stop("Playwright installed successfully!");
+		return true;
+	} catch (fallbackError) {
+		s.stop("Failed to install Playwright");
+		p.log.error(`Installation failed: ${fallbackError}`);
+		return false;
+	}
+}
+async function ensurePlaywrightInstalled() {
+	if (await checkPlaywrightInstallation()) return true;
+	p.log.warn("Playwright driver selected but Playwright is not installed.");
+	if (!await promptPlaywrightInstall()) {
+		p.log.error("Cannot proceed with Playwright driver without Playwright installed.");
+		return false;
+	}
+	return true;
+}
+async function isUseChromeSupported() {
+	try {
+		const { PlaywrightCrawler } = await import("crawlee");
+		const crawler = new PlaywrightCrawler({
+			launchContext: { useChrome: true },
+			requestHandler: async () => {},
+			maxRequestsPerCrawl: 1
+		});
+		const page = await crawler.browserPool.newPage();
+		await page.evaluate(() => {
+			return window.navigator.userAgent;
+		});
+		await page.close();
+		await crawler.browserPool.closeAllBrowsers();
+		crawler.stop();
+		return true;
+	} catch {}
+	return false;
+}
+//#endregion
+export { ensurePlaywrightInstalled, isUseChromeSupported };

package/dist/cli.mjs

@@ -1,64 +1,17 @@
 import { n as parseUrlPattern, r as validateGlobPattern, t as crawlAndGenerate } from "./_chunks/crawl.mjs";
 import { accessSync, constants, mkdirSync, readFileSync, unlinkSync, writeFileSync } from "node:fs";
 import * as p from "@clack/prompts";
-import { PlaywrightCrawler } from "crawlee";
 import { dirname, join, resolve } from "pathe";
 import { withHttps } from "ufo";
 import { fileURLToPath } from "node:url";
-import { addDependency } from "nypm";
-//#region src/playwright-utils.ts
-async function checkPlaywrightInstallation() {
-	try {
-		await import("playwright");
-		return true;
-	} catch {
-		return false;
-	}
-}
-async function promptPlaywrightInstall() {
-	const shouldInstall = await p.confirm({
-		message: "Playwright is required for the Playwright driver. Install it now?",
-		initialValue: true
+import { loadConfig } from "c12";
+//#region src/config.ts
+async function loadMdreamConfig(cwd) {
+	const { config } = await loadConfig({
+		name: "mdream",
+		cwd
 	});
-	if (p.isCancel(shouldInstall) || !shouldInstall) return false;
-	const s = p.spinner();
-	s.start("Installing Playwright globally...");
-	try {
-		await addDependency("playwright", { global: true });
-		s.stop("Playwright installed successfully!");
-		return true;
-	} catch (fallbackError) {
-		s.stop("Failed to install Playwright");
-		p.log.error(`Installation failed: ${fallbackError}`);
-		return false;
-	}
-}
-async function ensurePlaywrightInstalled() {
-	if (await checkPlaywrightInstallation()) return true;
-	p.log.warn("Playwright driver selected but Playwright is not installed.");
-	if (!await promptPlaywrightInstall()) {
-		p.log.error("Cannot proceed with Playwright driver without Playwright installed.");
-		return false;
-	}
-	return true;
-}
-async function isUseChromeSupported() {
-	try {
-		const crawler = new PlaywrightCrawler({
-			launchContext: { useChrome: true },
-			requestHandler: async () => {},
-			maxRequestsPerCrawl: 1
-		});
-		const page = await crawler.browserPool.newPage();
-		await page.evaluate(() => {
-			return window.navigator.userAgent;
-		});
-		await page.close();
-		await crawler.browserPool.closeAllBrowsers();
-		crawler.stop();
-		return true;
-	} catch {}
-	return false;
+	return config || {};
 }
 //#endregion
 //#region src/cli.ts
@@ -225,11 +178,17 @@ async function interactiveCrawl() {
 		skipSitemap: advancedOptions.skipSitemap
 	};
 }
-async function showCrawlResults(successful, failed, outputDir, generatedFiles, durationSeconds) {
+async function showCrawlResults(successful, failed, outputDir, generatedFiles, durationSeconds, latency) {
 	const messages = [];
 	const durationStr = `${durationSeconds.toFixed(1)}s`;
-	const stats = failed > 0 ? `${successful} pages, ${failed} failed` : `${successful} pages`;
-	messages.push(`📄 ${stats} • ⏱️  ${durationStr}`);
+	messages.push(`📄 ${successful} pages \u00B7 ⏱️  ${durationStr}`);
+	if (failed > 0) messages.push(`⚠️  ${failed} failed`);
+	if (latency && latency.count > 0) {
+		const avg = Math.round(latency.total / latency.count);
+		const min = latency.min === Infinity ? 0 : Math.round(latency.min);
+		const max = Math.round(latency.max);
+		messages.push(`🏓 avg ${avg}ms \u00B7 min ${min}ms \u00B7 max ${max}ms`);
+	}
 	messages.push(`📦 ${generatedFiles.join(", ")}`);
 	messages.push(`📁 ${outputDir}`);
 	p.note(messages.join("\n"), "✅ Complete");
@@ -249,7 +208,8 @@ Usage:
 Options:
   -u, --url <url>              Website URL to crawl
   -o, --output <dir>           Output directory (default: output)
-  -d, --depth <number>         Crawl depth (default: 3)
+  -d, --depth <number>         Crawl depth, 0 for single page (default: 3)
+  --single-page                Only process the given URL(s), no crawling (alias for --depth 0)
   --driver <http|playwright>   Crawler driver (default: http)
   --artifacts <list>           Comma-separated list of artifacts: llms.txt,llms-full.txt,markdown (default: all)
   --origin <url>               Origin URL for resolving relative paths (overrides auto-detection)
@@ -259,6 +219,7 @@ Options:
   --crawl-delay <seconds>     Crawl delay in seconds
   --exclude <pattern>         Exclude URLs matching glob patterns (can be used multiple times)
   --skip-sitemap              Skip sitemap.xml and robots.txt discovery
+  --allow-subdomains          Crawl across subdomains of the same root domain
   -v, --verbose               Enable verbose logging
   -h, --help                  Show this help message
   --version                   Show version number
@@ -271,6 +232,7 @@ Examples:
   @mdream/crawl -u example.com --exclude "*/admin/*" --exclude "*/api/*"
   @mdream/crawl -u example.com --verbose
   @mdream/crawl -u example.com --skip-sitemap
+  @mdream/crawl -u example.com --driver playwright --single-page
 `);
 		process.exit(0);
 	}
@@ -324,10 +286,10 @@ Examples:
 			process.exit(1);
 		}
 	}
-	const depthStr = getArgValue("--depth") || getArgValue("-d") || "3";
-	const depth = Number.parseInt(depthStr);
-	if (Number.isNaN(depth) || depth < 1 || depth > 10) {
-		p.log.error("Error: Depth must be between 1 and 10");
+	const depthStr = args.includes("--single-page") ? "0" : getArgValue("--depth") || getArgValue("-d") || "3";
+	const depth = Number(depthStr);
+	if (!Number.isInteger(depth) || depth < 0 || depth > 10) {
+		p.log.error("Error: Depth must be an integer between 0 and 10");
 		process.exit(1);
 	}
 	const driver = getArgValue("--driver");
@@ -381,13 +343,14 @@ Examples:
 	const patterns = [parsed];
 	const verbose = args.includes("--verbose") || args.includes("-v");
 	const skipSitemap = args.includes("--skip-sitemap");
+	const allowSubdomains = args.includes("--allow-subdomains");
 	if (skipSitemap && parsed.isGlob) p.log.warn("Warning: Using --skip-sitemap with glob URLs may not discover all matching pages.");
 	return {
 		urls: [url],
 		outputDir: resolve(getArgValue("--output") || getArgValue("-o") || "output"),
 		driver: driver || "http",
 		maxRequestsPerCrawl: Number.parseInt(maxPagesStr || String(Number.MAX_SAFE_INTEGER)),
-		followLinks: true,
+		followLinks: depth > 0,
 		maxDepth: depth,
 		generateLlmsTxt: artifacts.includes("llms.txt"),
 		generateLlmsFullTxt: artifacts.includes("llms-full.txt"),
@@ -399,14 +362,28 @@ Examples:
 		crawlDelay: crawlDelayStr ? Number.parseInt(crawlDelayStr) : void 0,
 		exclude: excludePatterns.length > 0 ? excludePatterns : void 0,
 		verbose,
-		skipSitemap
+		skipSitemap,
+		allowSubdomains
 	};
 }
 async function main() {
 	const cliOptions = parseCliArgs();
+	const fileConfig = await loadMdreamConfig();
 	let options;
 	if (cliOptions) {
-		options = cliOptions;
+		const configExclude = fileConfig.exclude || [];
+		const cliExclude = cliOptions.exclude || [];
+		options = {
+			...cliOptions,
+			driver: cliOptions.driver || fileConfig.driver || "http",
+			maxDepth: cliOptions.maxDepth ?? fileConfig.maxDepth,
+			crawlDelay: cliOptions.crawlDelay ?? fileConfig.crawlDelay,
+			skipSitemap: cliOptions.skipSitemap || fileConfig.skipSitemap || false,
+			allowSubdomains: cliOptions.allowSubdomains || fileConfig.allowSubdomains || false,
+			verbose: cliOptions.verbose || fileConfig.verbose || false,
+			exclude: configExclude.length > 0 || cliExclude.length > 0 ? [...configExclude, ...cliExclude] : void 0,
+			hooks: fileConfig.hooks
+		};
 		p.intro(`☁️  mdream v${version}`);
 		const formats = [];
 		if (options.generateLlmsTxt) formats.push("llms.txt");
@@ -420,6 +397,7 @@ async function main() {
 			`Formats: ${formats.join(", ")}`,
 			options.exclude && options.exclude.length > 0 && `Exclude: ${options.exclude.join(", ")}`,
 			options.skipSitemap && `Skip sitemap: Yes`,
+			options.allowSubdomains && `Allow subdomains: Yes`,
 			options.verbose && `Verbose: Enabled`
 		].filter(Boolean);
 		p.note(summary.join("\n"), "Configuration");
@@ -431,36 +409,46 @@ async function main() {
 		if (permCheck.error?.includes("Permission denied")) p.log.info("Tip: Try running with elevated privileges (e.g., sudo) or change the output directory permissions.");
 		process.exit(1);
 	}
-	if (options.driver === "playwright") if (await isUseChromeSupported()) {
-		options.useChrome = true;
-		p.log.info("System Chrome detected and enabled.");
-	} else {
-		if (!await ensurePlaywrightInstalled()) {
-			p.log.error("Cannot proceed without Playwright. Please install it manually or use the HTTP driver instead.");
+	if (options.driver === "playwright") {
+		try {
+			await import("crawlee");
+		} catch {
+			p.log.error("The Playwright driver requires crawlee. Install it with: npm install crawlee");
 			process.exit(1);
 		}
-		p.log.info("Using global playwright instance.");
+		const { ensurePlaywrightInstalled, isUseChromeSupported } = await import("./_chunks/playwright-utils.mjs");
+		if (await isUseChromeSupported()) {
+			options.useChrome = true;
+			p.log.info("System Chrome detected and enabled.");
+		} else {
+			if (!await ensurePlaywrightInstalled()) {
+				p.log.error("Cannot proceed without Playwright. Please install it manually or use the HTTP driver instead.");
+				process.exit(1);
+			}
+			p.log.info("Using global playwright instance.");
+		}
 	}
 	const s = p.spinner();
-	s.start("Starting crawl...");
+	s.start("Discovering sitemaps");
 	const startTime = Date.now();
+	let crawlStartTime = 0;
+	let lastProgress;
 	const results = await crawlAndGenerate(options, (progress) => {
-		if (progress.sitemap.status === "discovering") s.message("Discovering sitemaps...");
+		lastProgress = progress;
+		if (progress.sitemap.status === "discovering") s.message("Discovering sitemaps");
 		else if (progress.sitemap.status === "processing") s.message(`Processing sitemap... Found ${progress.sitemap.found} URLs`);
 		else if (progress.crawling.status === "processing") {
-			const processedCount = progress.crawling.processed;
-			const totalCount = progress.crawling.total;
-			const currentUrl = progress.crawling.currentUrl;
-			if (currentUrl) {
-				const shortUrl = currentUrl.length > 60 ? `${currentUrl.substring(0, 57)}...` : currentUrl;
-				if (processedCount > totalCount) s.message(`Crawling ${processedCount}: ${shortUrl}`);
-				else s.message(`Crawling ${processedCount}/${totalCount}: ${shortUrl}`);
-			} else if (processedCount > totalCount) s.message(`Crawling... ${processedCount} pages`);
-			else s.message(`Crawling... ${processedCount}/${totalCount} pages`);
-		} else if (progress.generation.status === "generating") {
-			const current = progress.generation.current || "Generating files";
-			s.message(current);
-		}
+			if (!crawlStartTime) crawlStartTime = Date.now();
+			const processed = progress.crawling.processed;
+			const total = progress.crawling.total;
+			const failed = progress.crawling.failed;
+			const elapsed = (Date.now() - crawlStartTime) / 1e3;
+			const rate = elapsed > .1 ? Math.round(processed / elapsed) : 0;
+			let msg = processed > total ? `Crawling ${processed} pages` : `Crawling ${processed}/${total}`;
+			if (rate > 0) msg += ` \u00B7 ${rate}/s`;
+			if (failed > 0) msg += ` \u00B7 ${failed} failed`;
+			s.message(msg);
+		} else if (progress.generation.status === "generating") s.message(progress.generation.current || "Generating files");
 	});
 	s.stop();
 	const durationSeconds = (Date.now() - startTime) / 1e3;
@@ -484,11 +472,13 @@ async function main() {
 		if (options.generateLlmsFullTxt) generatedFiles.push("llms-full.txt");
 		if (options.generateIndividualMd) generatedFiles.push(`${successful} MD files`);
 	}
-	await showCrawlResults(successful, failed, options.outputDir, generatedFiles, durationSeconds);
+	await showCrawlResults(successful, failed, options.outputDir, generatedFiles, durationSeconds, lastProgress?.crawling.latency);
 	process.exit(0);
 }
 main().catch((error) => {
-	p.log.error(`Unexpected error: ${error}`);
+	const msg = error instanceof Error ? error.message : String(error);
+	if (msg.includes("wmic") || msg.includes("ENOENT") && process.platform === "win32") p.log.error("Crawlee failed because wmic.exe is not available on this system. Windows 11 removed wmic.exe, which older crawlee versions depend on for memory monitoring.\nFix: upgrade crawlee to >=3.16.0 or switch to the HTTP driver (--driver http).");
+	else p.log.error(`Unexpected error: ${msg}`);
 	process.exit(1);
 });
 //#endregion

package/dist/index.d.mts

@@ -6,6 +6,22 @@ interface PageData {
   metadata: PageMetadata;
   origin: string;
 }
+interface CrawlHooks {
+  'crawl:url': (ctx: {
+    url: string;
+    skip: boolean;
+  }) => void | Promise<void>;
+  'crawl:page': (page: PageData) => void | Promise<void>;
+  'crawl:content': (ctx: {
+    url: string;
+    title: string;
+    content: string;
+    filePath: string;
+  }) => void | Promise<void>;
+  'crawl:done': (ctx: {
+    results: CrawlResult[];
+  }) => void | Promise<void>;
+}
 interface CrawlOptions {
   urls: string[];
   outputDir: string;
@@ -26,8 +42,23 @@ interface CrawlOptions {
   descriptionOverride?: string;
   verbose?: boolean;
   skipSitemap?: boolean;
+  allowSubdomains?: boolean;
+  hooks?: Partial<{ [K in keyof CrawlHooks]: CrawlHooks[K] | CrawlHooks[K][] }>;
   onPage?: (page: PageData) => Promise<void> | void;
 }
+interface MdreamCrawlConfig {
+  exclude?: string[];
+  driver?: 'http' | 'playwright';
+  maxDepth?: number;
+  maxPages?: number;
+  crawlDelay?: number;
+  skipSitemap?: boolean;
+  allowSubdomains?: boolean;
+  verbose?: boolean;
+  artifacts?: ('llms.txt' | 'llms-full.txt' | 'markdown')[];
+  hooks?: Partial<{ [K in keyof CrawlHooks]: CrawlHooks[K] | CrawlHooks[K][] }>;
+}
+declare function defineConfig(config: MdreamCrawlConfig): MdreamCrawlConfig;
 interface ParsedUrlPattern {
   baseUrl: string;
   pattern: string;
@@ -63,7 +94,14 @@ interface CrawlProgress {
     status: 'starting' | 'processing' | 'completed';
     total: number;
     processed: number;
-    currentUrl?: string;
+    failed: number;
+    currentUrl?: string; /** Page fetch latency stats in ms */
+    latency: {
+      total: number;
+      min: number;
+      max: number;
+      count: number;
+    };
   };
   generation: {
     status: 'idle' | 'generating' | 'completed';
@@ -72,4 +110,4 @@ interface CrawlProgress {
 }
 declare function crawlAndGenerate(options: CrawlOptions, onProgress?: (progress: CrawlProgress) => void): Promise<CrawlResult[]>;
 //#endregion
-export { type CrawlOptions, type CrawlResult, type PageData, crawlAndGenerate };
+export { type CrawlHooks, type CrawlOptions, type CrawlResult, type MdreamCrawlConfig, type PageData, crawlAndGenerate, defineConfig };

package/dist/index.mjs

@@ -1,2 +1,7 @@
 import { t as crawlAndGenerate } from "./_chunks/crawl.mjs";
-export { crawlAndGenerate };
+//#region src/types.ts
+function defineConfig(config) {
+	return config;
+}
+//#endregion
+export { crawlAndGenerate, defineConfig };

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@mdream/crawl",
   "type": "module",
-  "version": "1.0.0-beta.9",
+  "version": "1.0.1",
   "description": "Mdream Crawl generates comprehensive llms.txt artifacts from a single URL, using mdream to convert HTML to Markdown.",
   "author": {
     "name": "Harlan Wilton",
@@ -42,22 +42,29 @@
     "dist"
   ],
   "peerDependencies": {
+    "crawlee": "^3.16.0",
     "playwright": "^1.53.2"
   },
   "peerDependenciesMeta": {
+    "crawlee": {
+      "optional": true
+    },
     "playwright": {
       "optional": true
     }
   },
   "dependencies": {
     "@clack/prompts": "^1.1.0",
-    "crawlee": "^3.16.0",
+    "c12": "^3.0.4",
+    "hookable": "^5.5.3",
     "nypm": "^0.6.5",
+    "ofetch": "^1.5.1",
     "pathe": "^2.0.3",
     "picomatch": "^4.0.3",
+    "tldts": "^7.0.26",
     "ufo": "^1.6.3",
-    "@mdream/js": "1.0.0-beta.9",
-    "mdream": "1.0.0-beta.9"
+    "@mdream/js": "1.0.1",
+    "mdream": "1.0.1"
   },
   "devDependencies": {
     "@types/picomatch": "^4.0.2"