@mdream/crawl 0.8.4 → 0.9.0

package/dist/_chunks/{crawl-DYXGzu7W.mjs → crawl-BtuYX2_u.mjs}

@@ -5,24 +5,10 @@ import { HttpCrawler, PlaywrightCrawler, log, purgeDefaultStorages } from "crawl
 import { generateLlmsTxtArtifacts, htmlToMarkdown } from "mdream";
 import { withMinimalPreset } from "mdream/preset/minimal";
 import { dirname, join, normalize, resolve } from "pathe";
+import { withHttps } from "ufo";
 import picomatch from "picomatch";
 import { extractionPlugin } from "mdream/plugins";
 
-//#region ../../node_modules/.pnpm/ufo@1.6.1/node_modules/ufo/dist/index.mjs
-const r = String.fromCharCode;
-const PROTOCOL_REGEX = /^[\s\w\0+.-]{2,}:([/\\]{2})?/;
-function withHttps(input) {
-	return withProtocol(input, "https://");
-}
-function withProtocol(input, protocol) {
-	let match = input.match(PROTOCOL_REGEX);
-	if (!match) match = input.match(/^\/{2,}/);
-	if (!match) return protocol + input;
-	return protocol + input.slice(match[0].length);
-}
-const protocolRelative = Symbol.for("ufo:protocolRelative");
-
-//#endregion
 //#region src/glob-utils.ts
 /**
 * Parse a URL that may contain glob patterns
@@ -467,14 +453,14 @@ async function crawlAndGenerate(options, onProgress) {
 	await crawler.run(initialRequests);
 	progress.crawling.status = "completed";
 	onProgress?.(progress);
-	if (results.some((r$1) => r$1.success)) {
+	if (results.some((r) => r.success)) {
 		progress.generation.status = "generating";
 		onProgress?.(progress);
-		const successfulResults = results.filter((r$1) => r$1.success);
+		const successfulResults = results.filter((r) => r.success);
 		const firstUrl = new URL(withHttps(urls[0]));
 		const origin$1 = firstUrl.origin;
-		const homePageResult = successfulResults.find((r$1) => {
-			const resultUrl = new URL(withHttps(r$1.url));
+		const homePageResult = successfulResults.find((r) => {
+			const resultUrl = new URL(withHttps(r.url));
 			return resultUrl.href === origin$1 || resultUrl.href === `${origin$1}/`;
 		});
 		const siteName = siteNameOverride || homePageResult?.metadata?.title || homePageResult?.title || firstUrl.hostname;
@@ -528,4 +514,4 @@ async function crawlAndGenerate(options, onProgress) {
 }
 
 //#endregion
-export { crawlAndGenerate, parseUrlPattern, validateGlobPattern, withHttps };
+export { crawlAndGenerate, parseUrlPattern, validateGlobPattern };

package/dist/cli.mjs

@@ -1,9 +1,10 @@
-import { crawlAndGenerate, parseUrlPattern, validateGlobPattern, withHttps } from "./_chunks/crawl-DYXGzu7W.mjs";
-import { readFileSync } from "node:fs";
+import { crawlAndGenerate, parseUrlPattern, validateGlobPattern } from "./_chunks/crawl-BtuYX2_u.mjs";
+import { accessSync, constants, mkdirSync, readFileSync, unlinkSync, writeFileSync } from "node:fs";
 import * as p$1 from "@clack/prompts";
 import * as p from "@clack/prompts";
 import { PlaywrightCrawler } from "crawlee";
 import { dirname, join, resolve } from "pathe";
+import { withHttps } from "ufo";
 import { fileURLToPath } from "node:url";
 import { addDependency } from "nypm";
 
@@ -70,6 +71,38 @@ const __dirname = dirname(fileURLToPath(import.meta.url));
 const packageJsonPath = join(__dirname, "..", "package.json");
 const packageJson = JSON.parse(readFileSync(packageJsonPath, "utf-8"));
 const version = packageJson.version;
+function checkOutputDirectoryPermissions(outputDir) {
+	try {
+		mkdirSync(outputDir, { recursive: true });
+		accessSync(outputDir, constants.W_OK);
+		const testFile = join(outputDir, ".mdream-test");
+		try {
+			writeFileSync(testFile, "test");
+			unlinkSync(testFile);
+		} catch (err) {
+			return {
+				success: false,
+				error: `Cannot write to output directory: ${err instanceof Error ? err.message : "Unknown error"}`
+			};
+		}
+		return { success: true };
+	} catch (err) {
+		if (err instanceof Error) {
+			if (err.message.includes("EACCES")) return {
+				success: false,
+				error: `Permission denied: Cannot write to output directory '${outputDir}'. Please check permissions or run with appropriate privileges.`
+			};
+			return {
+				success: false,
+				error: `Failed to access output directory: ${err.message}`
+			};
+		}
+		return {
+			success: false,
+			error: "Failed to access output directory"
+		};
+	}
+}
 async function interactiveCrawl() {
 	console.clear();
 	p.intro(`☁️  @mdream/crawl v${version}`);
@@ -108,61 +141,44 @@ async function interactiveCrawl() {
 		return null;
 	}
 	const outputDir = "output";
-	const crawlerOptions = await p.group({
-		driver: () => p.select({
-			message: "Select crawler driver:",
-			options: [{
-				value: "http",
-				label: "HTTP Crawler (Fast, for static content)",
-				hint: "Recommended"
-			}, {
-				value: "playwright",
-				label: "Playwright (Slower, supports JavaScript)"
-			}],
-			initialValue: "http"
-		}),
-		maxDepth: () => p.text({
-			message: "Clicks to page (crawl depth):",
-			placeholder: "3",
-			defaultValue: "3",
-			validate: (value) => {
-				const num = Number.parseInt(value);
-				if (Number.isNaN(num) || num < 1 || num > 10) return "Depth must be between 1 and 10";
-			}
-		})
-	}, { onCancel: () => {
+	const crawlerOptions = await p.group({ driver: () => p.select({
+		message: "Select crawler driver:",
+		options: [{
+			value: "http",
+			label: "HTTP Crawler (Fast, for static content)",
+			hint: "Recommended"
+		}, {
+			value: "playwright",
+			label: "Playwright (Slower, supports JavaScript)"
+		}],
+		initialValue: "http"
+	}) }, { onCancel: () => {
 		p.cancel("Operation cancelled.");
 		process.exit(0);
 	} });
-	const advancedOptions = await p.group({
-		outputFormats: () => p.multiselect({
-			message: "Select output formats:",
-			options: [
-				{
-					value: "llms.txt",
-					label: "llms.txt (basic format)",
-					hint: "Recommended"
-				},
-				{
-					value: "llms-full.txt",
-					label: "llms-full.txt (extended format)"
-				},
-				{
-					value: "markdown",
-					label: "Individual Markdown files"
-				}
-			],
-			initialValues: [
-				"llms.txt",
-				"llms-full.txt",
-				"markdown"
-			]
-		}),
-		verbose: () => p.confirm({
-			message: "Enable verbose logging?",
-			initialValue: false
-		})
-	}, { onCancel: () => {
+	const advancedOptions = await p.group({ outputFormats: () => p.multiselect({
+		message: "Select output formats:",
+		options: [
+			{
+				value: "llms.txt",
+				label: "llms.txt (basic format)",
+				hint: "Recommended"
+			},
+			{
+				value: "llms-full.txt",
+				label: "llms-full.txt (extended format)"
+			},
+			{
+				value: "markdown",
+				label: "Individual Markdown files"
+			}
+		],
+		initialValues: [
+			"llms.txt",
+			"llms-full.txt",
+			"markdown"
+		]
+	}) }, { onCancel: () => {
 		p.cancel("Operation cancelled.");
 		process.exit(0);
 	} });
@@ -188,34 +204,25 @@ async function interactiveCrawl() {
 		`Output: ${outputDir}`,
 		`Driver: ${crawlerOptions.driver}`,
 		`Max pages: Unlimited`,
-		`Follow links: Yes (depth ${crawlerOptions.maxDepth})`,
+		`Follow links: Yes (depth 3)`,
 		`Output formats: ${outputFormats.join(", ")}`,
 		`Sitemap discovery: Automatic`,
-		inferredOrigin && `Origin: ${inferredOrigin}`,
-		advancedOptions.verbose && `Verbose logging: Enabled`
+		inferredOrigin && `Origin: ${inferredOrigin}`
 	].filter(Boolean);
 	p.note(summary.join("\n"), "Crawl Configuration");
-	const shouldProceed = await p.confirm({
-		message: "Start crawling?",
-		initialValue: true
-	});
-	if (p.isCancel(shouldProceed) || !shouldProceed) {
-		p.cancel("Crawl cancelled.");
-		return null;
-	}
 	return {
 		urls,
 		outputDir: resolve(outputDir),
 		driver: crawlerOptions.driver,
 		maxRequestsPerCrawl: Number.MAX_SAFE_INTEGER,
 		followLinks: true,
-		maxDepth: Number.parseInt(crawlerOptions.maxDepth),
 		generateLlmsTxt: advancedOptions.outputFormats.includes("llms.txt"),
 		generateLlmsFullTxt: advancedOptions.outputFormats.includes("llms-full.txt"),
 		generateIndividualMd: advancedOptions.outputFormats.includes("markdown"),
 		origin: inferredOrigin,
 		globPatterns,
-		verbose: advancedOptions.verbose
+		verbose: false,
+		maxDepth: 3
 	};
 }
 async function showCrawlResults(successful, failed, outputDir, generatedFiles, durationSeconds) {
@@ -412,6 +419,12 @@ async function main() {
 		p.note(summary.join("\n"), "Configuration");
 	} else options = await interactiveCrawl();
 	if (!options) process.exit(0);
+	const permCheck = checkOutputDirectoryPermissions(options.outputDir);
+	if (!permCheck.success) {
+		p.log.error(permCheck.error);
+		if (permCheck.error?.includes("Permission denied")) p.log.info("Tip: Try running with elevated privileges (e.g., sudo) or change the output directory permissions.");
+		process.exit(1);
+	}
 	if (options.driver === "playwright") {
 		const chromeSupported = await isUseChromeSupported();
 		if (chromeSupported) {

package/dist/index.mjs

@@ -1,4 +1,4 @@
-import { crawlAndGenerate } from "./_chunks/crawl-DYXGzu7W.mjs";
+import { crawlAndGenerate } from "./_chunks/crawl-BtuYX2_u.mjs";
 import { writeFile } from "node:fs/promises";
 import { basename, sep } from "pathe";
 

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@mdream/crawl",
   "type": "module",
-  "version": "0.8.4",
+  "version": "0.9.0",
   "description": "Mdream Crawl generates comprehensive llms.txt artifacts from a single URL, using mdream to convert HTML to Markdown.",
   "author": {
     "name": "Harlan Wilton",
@@ -46,11 +46,12 @@
   },
   "dependencies": {
     "@clack/prompts": "^0.11.0",
-    "crawlee": "^3.13.10",
+    "crawlee": "^3.14.0",
     "nypm": "^0.6.0",
     "pathe": "^2.0.3",
     "picomatch": "^4.0.3",
-    "mdream": "0.8.4"
+    "ufo": "^1.6.1",
+    "mdream": "0.9.0"
   },
   "devDependencies": {
     "@types/picomatch": "^4.0.1"