@mdream/crawl 0.8.2 → 0.8.4

package/dist/_chunks/{crawl-BwURA9nQ.mjs → crawl-DYXGzu7W.mjs}

@@ -194,7 +194,7 @@ async function loadSitemapWithoutRetries(sitemapUrl) {
 	return urls;
 }
 async function crawlAndGenerate(options, onProgress) {
-	const { urls, outputDir: rawOutputDir, maxRequestsPerCrawl = Number.MAX_SAFE_INTEGER, generateLlmsTxt = true, generateLlmsFullTxt = false, generateIndividualMd = true, origin, driver = "http", followLinks = false, maxDepth = 1, globPatterns = [], crawlDelay, exclude = [], siteNameOverride, descriptionOverride, verbose = false } = options;
+	const { urls, outputDir: rawOutputDir, maxRequestsPerCrawl = Number.MAX_SAFE_INTEGER, generateLlmsTxt = true, generateLlmsFullTxt = false, generateIndividualMd = true, origin, driver = "http", useChrome, followLinks = false, maxDepth = 1, globPatterns = [], crawlDelay, exclude = [], siteNameOverride, descriptionOverride, verbose = false } = options;
 	const outputDir = resolve(normalize(rawOutputDir));
 	if (verbose) log.setLevel(log.LEVELS.INFO);
 	else log.setLevel(log.LEVELS.OFF);
@@ -449,8 +449,14 @@ async function crawlAndGenerate(options, onProgress) {
 		respectRobotsTxtFile: true
 	};
 	if (crawlDelay) crawlerOptions.requestHandlerTimeoutSecs = crawlDelay;
-	if (driver === "playwright") crawler = new PlaywrightCrawler(crawlerOptions);
-	else crawler = new HttpCrawler(crawlerOptions);
+	if (driver === "playwright") {
+		const playwrightOptions = crawlerOptions;
+		if (useChrome) playwrightOptions.launchContext = {
+			...playwrightOptions.launchContext,
+			useChrome
+		};
+		crawler = new PlaywrightCrawler(playwrightOptions);
+	} else crawler = new HttpCrawler(crawlerOptions);
 	const initialRequests = startingUrls.map((url) => ({
 		url,
 		userData: { depth: 0 }

package/dist/cli.mjs

@@ -1,7 +1,8 @@
-import { crawlAndGenerate, parseUrlPattern, validateGlobPattern, withHttps } from "./_chunks/crawl-BwURA9nQ.mjs";
+import { crawlAndGenerate, parseUrlPattern, validateGlobPattern, withHttps } from "./_chunks/crawl-DYXGzu7W.mjs";
 import { readFileSync } from "node:fs";
 import * as p$1 from "@clack/prompts";
 import * as p from "@clack/prompts";
+import { PlaywrightCrawler } from "crawlee";
 import { dirname, join, resolve } from "pathe";
 import { fileURLToPath } from "node:url";
 import { addDependency } from "nypm";
@@ -22,21 +23,15 @@ async function promptPlaywrightInstall() {
 	});
 	if (p$1.isCancel(shouldInstall) || !shouldInstall) return false;
 	const s = p$1.spinner();
-	s.start("Installing Playwright...");
+	s.start("Installing Playwright globally...");
 	try {
-		await addDependency("playwright", { workspace: true });
+		await addDependency("playwright", { global: true });
 		s.stop("Playwright installed successfully!");
 		return true;
-	} catch {
-		try {
-			await addDependency("playwright");
-			s.stop("Playwright installed successfully!");
-			return true;
-		} catch (fallbackError) {
-			s.stop("Failed to install Playwright");
-			p$1.log.error(`Installation failed: ${fallbackError}`);
-			return false;
-		}
+	} catch (fallbackError) {
+		s.stop("Failed to install Playwright");
+		p$1.log.error(`Installation failed: ${fallbackError}`);
+		return false;
 	}
 }
 async function ensurePlaywrightInstalled() {
@@ -50,6 +45,24 @@ async function ensurePlaywrightInstalled() {
 	}
 	return true;
 }
+async function isUseChromeSupported() {
+	try {
+		const crawler = new PlaywrightCrawler({
+			launchContext: { useChrome: true },
+			requestHandler: async () => {},
+			maxRequestsPerCrawl: 1
+		});
+		const page = await crawler.browserPool.newPage();
+		await page.evaluate(() => {
+			return window.navigator.userAgent;
+		});
+		await page.close();
+		await crawler.browserPool.closeAllBrowsers();
+		crawler.stop();
+		return true;
+	} catch {}
+	return false;
+}
 
 //#endregion
 //#region src/cli.ts
@@ -400,10 +413,17 @@ async function main() {
 	} else options = await interactiveCrawl();
 	if (!options) process.exit(0);
 	if (options.driver === "playwright") {
-		const playwrightInstalled = await ensurePlaywrightInstalled();
-		if (!playwrightInstalled) {
-			p.log.error("Cannot proceed without Playwright. Please install it manually or use the HTTP driver instead.");
-			process.exit(1);
+		const chromeSupported = await isUseChromeSupported();
+		if (chromeSupported) {
+			options.useChrome = true;
+			p.log.info("System Chrome detected and enabled.");
+		} else {
+			const playwrightInstalled = await ensurePlaywrightInstalled();
+			if (!playwrightInstalled) {
+				p.log.error("Cannot proceed without Playwright. Please install it manually or use the HTTP driver instead.");
+				process.exit(1);
+			}
+			p.log.info("Using global playwright instance.");
 		}
 	}
 	const s = p.spinner();
@@ -451,11 +471,8 @@ async function main() {
 		if (options.generateLlmsFullTxt) generatedFiles.push("llms-full.txt");
 		if (options.generateIndividualMd) generatedFiles.push(`${successful} MD files`);
 	}
-	if (!cliOptions) await showCrawlResults(successful, failed, options.outputDir, generatedFiles, durationSeconds);
-	else {
-		await showCrawlResults(successful, failed, options.outputDir, generatedFiles, durationSeconds);
-		if (successful === 0) process.exit(1);
-	}
+	await showCrawlResults(successful, failed, options.outputDir, generatedFiles, durationSeconds);
+	process.exit(0);
 }
 main().catch((error) => {
 	p.log.error(`Unexpected error: ${error}`);

package/dist/index.d.mts

@@ -9,6 +9,7 @@ interface CrawlOptions {
   origin?: string;
   chunkSize?: number;
   driver?: 'http' | 'playwright';
+  useChrome?: boolean;
   followLinks?: boolean;
   maxDepth?: number;
   globPatterns?: ParsedUrlPattern[];

package/dist/index.mjs

@@ -1,4 +1,4 @@
-import { crawlAndGenerate } from "./_chunks/crawl-BwURA9nQ.mjs";
+import { crawlAndGenerate } from "./_chunks/crawl-DYXGzu7W.mjs";
 import { writeFile } from "node:fs/promises";
 import { basename, sep } from "pathe";
 

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@mdream/crawl",
   "type": "module",
-  "version": "0.8.2",
+  "version": "0.8.4",
   "description": "Mdream Crawl generates comprehensive llms.txt artifacts from a single URL, using mdream to convert HTML to Markdown.",
   "author": {
     "name": "Harlan Wilton",
@@ -50,7 +50,7 @@
     "nypm": "^0.6.0",
     "pathe": "^2.0.3",
     "picomatch": "^4.0.3",
-    "mdream": "0.8.2"
+    "mdream": "0.8.4"
   },
   "devDependencies": {
     "@types/picomatch": "^4.0.1"