@mdream/crawl 0.8.5 → 0.9.0

package/dist/cli.mjs

@@ -1,5 +1,5 @@
 import { crawlAndGenerate, parseUrlPattern, validateGlobPattern } from "./_chunks/crawl-BtuYX2_u.mjs";
-import { readFileSync } from "node:fs";
+import { accessSync, constants, mkdirSync, readFileSync, unlinkSync, writeFileSync } from "node:fs";
 import * as p$1 from "@clack/prompts";
 import * as p from "@clack/prompts";
 import { PlaywrightCrawler } from "crawlee";
@@ -71,6 +71,38 @@ const __dirname = dirname(fileURLToPath(import.meta.url));
 const packageJsonPath = join(__dirname, "..", "package.json");
 const packageJson = JSON.parse(readFileSync(packageJsonPath, "utf-8"));
 const version = packageJson.version;
+function checkOutputDirectoryPermissions(outputDir) {
+	try {
+		mkdirSync(outputDir, { recursive: true });
+		accessSync(outputDir, constants.W_OK);
+		const testFile = join(outputDir, ".mdream-test");
+		try {
+			writeFileSync(testFile, "test");
+			unlinkSync(testFile);
+		} catch (err) {
+			return {
+				success: false,
+				error: `Cannot write to output directory: ${err instanceof Error ? err.message : "Unknown error"}`
+			};
+		}
+		return { success: true };
+	} catch (err) {
+		if (err instanceof Error) {
+			if (err.message.includes("EACCES")) return {
+				success: false,
+				error: `Permission denied: Cannot write to output directory '${outputDir}'. Please check permissions or run with appropriate privileges.`
+			};
+			return {
+				success: false,
+				error: `Failed to access output directory: ${err.message}`
+			};
+		}
+		return {
+			success: false,
+			error: "Failed to access output directory"
+		};
+	}
+}
 async function interactiveCrawl() {
 	console.clear();
 	p.intro(`☁️  @mdream/crawl v${version}`);
@@ -109,61 +141,44 @@ async function interactiveCrawl() {
 		return null;
 	}
 	const outputDir = "output";
-	const crawlerOptions = await p.group({
-		driver: () => p.select({
-			message: "Select crawler driver:",
-			options: [{
-				value: "http",
-				label: "HTTP Crawler (Fast, for static content)",
-				hint: "Recommended"
-			}, {
-				value: "playwright",
-				label: "Playwright (Slower, supports JavaScript)"
-			}],
-			initialValue: "http"
-		}),
-		maxDepth: () => p.text({
-			message: "Clicks to page (crawl depth):",
-			placeholder: "3",
-			defaultValue: "3",
-			validate: (value) => {
-				const num = Number.parseInt(value);
-				if (Number.isNaN(num) || num < 1 || num > 10) return "Depth must be between 1 and 10";
-			}
-		})
-	}, { onCancel: () => {
+	const crawlerOptions = await p.group({ driver: () => p.select({
+		message: "Select crawler driver:",
+		options: [{
+			value: "http",
+			label: "HTTP Crawler (Fast, for static content)",
+			hint: "Recommended"
+		}, {
+			value: "playwright",
+			label: "Playwright (Slower, supports JavaScript)"
+		}],
+		initialValue: "http"
+	}) }, { onCancel: () => {
 		p.cancel("Operation cancelled.");
 		process.exit(0);
 	} });
-	const advancedOptions = await p.group({
-		outputFormats: () => p.multiselect({
-			message: "Select output formats:",
-			options: [
-				{
-					value: "llms.txt",
-					label: "llms.txt (basic format)",
-					hint: "Recommended"
-				},
-				{
-					value: "llms-full.txt",
-					label: "llms-full.txt (extended format)"
-				},
-				{
-					value: "markdown",
-					label: "Individual Markdown files"
-				}
-			],
-			initialValues: [
-				"llms.txt",
-				"llms-full.txt",
-				"markdown"
-			]
-		}),
-		verbose: () => p.confirm({
-			message: "Enable verbose logging?",
-			initialValue: false
-		})
-	}, { onCancel: () => {
+	const advancedOptions = await p.group({ outputFormats: () => p.multiselect({
+		message: "Select output formats:",
+		options: [
+			{
+				value: "llms.txt",
+				label: "llms.txt (basic format)",
+				hint: "Recommended"
+			},
+			{
+				value: "llms-full.txt",
+				label: "llms-full.txt (extended format)"
+			},
+			{
+				value: "markdown",
+				label: "Individual Markdown files"
+			}
+		],
+		initialValues: [
+			"llms.txt",
+			"llms-full.txt",
+			"markdown"
+		]
+	}) }, { onCancel: () => {
 		p.cancel("Operation cancelled.");
 		process.exit(0);
 	} });
@@ -189,34 +204,25 @@ async function interactiveCrawl() {
 		`Output: ${outputDir}`,
 		`Driver: ${crawlerOptions.driver}`,
 		`Max pages: Unlimited`,
-		`Follow links: Yes (depth ${crawlerOptions.maxDepth})`,
+		`Follow links: Yes (depth 3)`,
 		`Output formats: ${outputFormats.join(", ")}`,
 		`Sitemap discovery: Automatic`,
-		inferredOrigin && `Origin: ${inferredOrigin}`,
-		advancedOptions.verbose && `Verbose logging: Enabled`
+		inferredOrigin && `Origin: ${inferredOrigin}`
 	].filter(Boolean);
 	p.note(summary.join("\n"), "Crawl Configuration");
-	const shouldProceed = await p.confirm({
-		message: "Start crawling?",
-		initialValue: true
-	});
-	if (p.isCancel(shouldProceed) || !shouldProceed) {
-		p.cancel("Crawl cancelled.");
-		return null;
-	}
 	return {
 		urls,
 		outputDir: resolve(outputDir),
 		driver: crawlerOptions.driver,
 		maxRequestsPerCrawl: Number.MAX_SAFE_INTEGER,
 		followLinks: true,
-		maxDepth: Number.parseInt(crawlerOptions.maxDepth),
 		generateLlmsTxt: advancedOptions.outputFormats.includes("llms.txt"),
 		generateLlmsFullTxt: advancedOptions.outputFormats.includes("llms-full.txt"),
 		generateIndividualMd: advancedOptions.outputFormats.includes("markdown"),
 		origin: inferredOrigin,
 		globPatterns,
-		verbose: advancedOptions.verbose
+		verbose: false,
+		maxDepth: 3
 	};
 }
 async function showCrawlResults(successful, failed, outputDir, generatedFiles, durationSeconds) {
@@ -413,6 +419,12 @@ async function main() {
 		p.note(summary.join("\n"), "Configuration");
 	} else options = await interactiveCrawl();
 	if (!options) process.exit(0);
+	const permCheck = checkOutputDirectoryPermissions(options.outputDir);
+	if (!permCheck.success) {
+		p.log.error(permCheck.error);
+		if (permCheck.error?.includes("Permission denied")) p.log.info("Tip: Try running with elevated privileges (e.g., sudo) or change the output directory permissions.");
+		process.exit(1);
+	}
 	if (options.driver === "playwright") {
 		const chromeSupported = await isUseChromeSupported();
 		if (chromeSupported) {

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@mdream/crawl",
   "type": "module",
-  "version": "0.8.5",
+  "version": "0.9.0",
   "description": "Mdream Crawl generates comprehensive llms.txt artifacts from a single URL, using mdream to convert HTML to Markdown.",
   "author": {
     "name": "Harlan Wilton",
@@ -46,12 +46,12 @@
   },
   "dependencies": {
     "@clack/prompts": "^0.11.0",
-    "crawlee": "^3.13.10",
+    "crawlee": "^3.14.0",
     "nypm": "^0.6.0",
     "pathe": "^2.0.3",
     "picomatch": "^4.0.3",
     "ufo": "^1.6.1",
-    "mdream": "0.8.5"
+    "mdream": "0.9.0"
   },
   "devDependencies": {
     "@types/picomatch": "^4.0.1"