@mdream/crawl 0.15.3 → 0.17.0

package/dist/_chunks/crawl.mjs

@@ -9,8 +9,9 @@ import { dirname, join, normalize, resolve } from "pathe";
 import { withHttps } from "ufo";
 import picomatch from "picomatch";
 import { extractionPlugin } from "mdream/plugins";
-
 //#region src/glob-utils.ts
+const GLOB_STRIP_TAIL_RE = /\*.*$/;
+const GLOB_CHAR_RE = /[*?[]/;
 /**
 * Parse a URL that may contain glob patterns
 * Example: https://nuxtseo.com/docs/** -> { baseUrl: "https://nuxtseo.com", pattern: "/docs/**", isGlob: true }
@@ -22,7 +23,7 @@ function parseUrlPattern(input) {
 		isGlob: false
 	};
 	try {
-		const urlWithoutGlob = (input.startsWith("http") ? input : `https://${input}`).replace(/\*.*$/, "");
+		const urlWithoutGlob = (input.startsWith("http") ? input : `https://${input}`).replace(GLOB_STRIP_TAIL_RE, "");
 		const url = new URL(urlWithoutGlob);
 		const baseUrl = `${url.protocol}//${url.host}`;
 		const patternStart = input.indexOf(url.host) + url.host.length;
@@ -61,7 +62,7 @@ function matchesGlobPattern(url, parsedPattern) {
 function getStartingUrl(parsedPattern) {
 	if (!parsedPattern.isGlob) return withHttps(parsedPattern.baseUrl);
 	const pattern = parsedPattern.pattern;
-	const firstGlobIndex = pattern.search(/[*?[]/);
+	const firstGlobIndex = pattern.search(GLOB_CHAR_RE);
 	if (firstGlobIndex === -1) return withHttps(parsedPattern.baseUrl + pattern);
 	const beforeGlob = pattern.substring(0, firstGlobIndex);
 	const lastSlash = beforeGlob.lastIndexOf("/");
@@ -100,7 +101,6 @@ function validateGlobPattern(pattern) {
 		return `Invalid glob pattern: ${error instanceof Error ? error.message : error}`;
 	}
 }
-
 //#endregion
 //#region src/metadata-extractor.ts
 function extractMetadata(html, url) {
@@ -155,9 +155,15 @@ function extractMetadata(html, url) {
 		})
 	};
 }
-
 //#endregion
 //#region src/crawl.ts
+const SITEMAP_INDEX_LOC_RE = /<sitemap[^>]*>.*?<loc>(.*?)<\/loc>.*?<\/sitemap>/gs;
+const SITEMAP_URL_LOC_RE = /<url[^>]*>.*?<loc>(.*?)<\/loc>.*?<\/url>/gs;
+const ROBOTS_SITEMAP_RE = /Sitemap:\s*(.*)/gi;
+const ROBOTS_SITEMAP_PREFIX_RE = /Sitemap:\s*/i;
+const URL_TRAILING_SLASH_RE = /\/$/;
+const URL_PATH_UNSAFE_CHARS_RE = /[^\w\-]/g;
+const FRONTMATTER_BLOCK_RE = /^---[^\n]*\n[\s\S]*?\n---[^\n]*\n?/;
 async function loadSitemapWithoutRetries(sitemapUrl) {
 	const controller = new AbortController();
 	const timeoutId = setTimeout(() => controller.abort(), 1e4);
@@ -170,11 +176,11 @@ async function loadSitemapWithoutRetries(sitemapUrl) {
 		if (!response.ok) throw new Error(`Sitemap not found: ${response.status}`);
 		const xmlContent = await response.text();
 		if (xmlContent.includes("<sitemapindex")) {
-			const sitemapIndexRegex = /<sitemap[^>]*>.*?<loc>(.*?)<\/loc>.*?<\/sitemap>/gs;
+			SITEMAP_INDEX_LOC_RE.lastIndex = 0;
 			const childSitemaps = [];
 			let match;
 			while (true) {
-				match = sitemapIndexRegex.exec(xmlContent);
+				match = SITEMAP_INDEX_LOC_RE.exec(xmlContent);
 				if (match === null) break;
 				let url = match[1];
 				if (url.startsWith("<![CDATA[") && url.endsWith("]]>")) url = url.slice(9, -3);
@@ -190,10 +196,10 @@ async function loadSitemapWithoutRetries(sitemapUrl) {
 			return allUrls;
 		} else {
 			const urls = [];
-			const urlRegex = /<url[^>]*>.*?<loc>(.*?)<\/loc>.*?<\/url>/gs;
+			SITEMAP_URL_LOC_RE.lastIndex = 0;
 			let match;
 			while (true) {
-				match = urlRegex.exec(xmlContent);
+				match = SITEMAP_URL_LOC_RE.exec(xmlContent);
 				if (match === null) break;
 				let url = match[1];
 				if (url.startsWith("<![CDATA[") && url.endsWith("]]>")) url = url.slice(9, -3);
@@ -247,24 +253,24 @@ async function crawlAndGenerate(options, onProgress) {
 				headers: { "User-Agent": "mdream-crawler/1.0" }
 			});
 			clearTimeout(robotsTimeoutId);
-		} catch (error) {
+		} catch {
 			clearTimeout(robotsTimeoutId);
 			robotsResponse = null;
 		}
 		if (robotsResponse?.ok) {
-			const sitemapMatches = (await robotsResponse.text()).match(/Sitemap:\s*(.*)/gi);
+			const sitemapMatches = (await robotsResponse.text()).match(ROBOTS_SITEMAP_RE);
 			if (sitemapMatches && sitemapMatches.length > 0) {
 				progress.sitemap.found = sitemapMatches.length;
 				progress.sitemap.status = "processing";
 				onProgress?.(progress);
-				const robotsSitemaps = sitemapMatches.map((match) => match.replace(/Sitemap:\s*/i, "").trim());
+				const robotsSitemaps = sitemapMatches.map((match) => match.replace(ROBOTS_SITEMAP_PREFIX_RE, "").trim());
 				for (const sitemapUrl of robotsSitemaps) try {
 					const robotsUrls = await loadSitemapWithoutRetries(sitemapUrl);
 					sitemapAttempts.push({
 						url: sitemapUrl,
 						success: true
 					});
-					if (patterns.some((p$1) => p$1.isGlob)) {
+					if (patterns.some((p) => p.isGlob)) {
 						const filteredUrls = robotsUrls.filter((url) => {
 							return !isUrlExcluded(url, exclude) && patterns.some((pattern) => matchesGlobPattern(url, pattern));
 						});
@@ -300,7 +306,7 @@ async function crawlAndGenerate(options, onProgress) {
 				url: mainSitemapUrl,
 				success: true
 			});
-			if (patterns.some((p$1) => p$1.isGlob)) {
+			if (patterns.some((p) => p.isGlob)) {
 				const filteredUrls = sitemapUrls.filter((url) => {
 					return !isUrlExcluded(url, exclude) && patterns.some((pattern) => matchesGlobPattern(url, pattern));
 				});
@@ -339,7 +345,7 @@ async function crawlAndGenerate(options, onProgress) {
 						url: sitemapUrl,
 						success: true
 					});
-					if (patterns.some((p$1) => p$1.isGlob)) {
+					if (patterns.some((p) => p.isGlob)) {
 						const filteredUrls = altUrls.filter((url) => {
 							return !isUrlExcluded(url, exclude) && patterns.some((pattern) => matchesGlobPattern(url, pattern));
 						});
@@ -360,11 +366,11 @@ async function crawlAndGenerate(options, onProgress) {
 							break;
 						}
 					}
-				} catch (error$1) {
+				} catch (error) {
 					sitemapAttempts.push({
 						url: sitemapUrl,
 						success: false,
-						error: error$1 instanceof Error ? error$1.message : "Unknown error"
+						error: error instanceof Error ? error.message : "Unknown error"
 					});
 				}
 			}
@@ -396,7 +402,7 @@ async function crawlAndGenerate(options, onProgress) {
 	const processedUrls = /* @__PURE__ */ new Set();
 	const shouldCrawlUrl = (url) => {
 		if (isUrlExcluded(url, exclude)) return false;
-		if (!patterns.some((p$1) => p$1.isGlob)) return true;
+		if (!patterns.some((p) => p.isGlob)) return true;
 		return patterns.some((pattern) => matchesGlobPattern(url, pattern));
 	};
 	const createRequestHandler = (crawlerType) => {
@@ -428,17 +434,17 @@ async function crawlAndGenerate(options, onProgress) {
 				origin: pageOrigin
 			});
 			let md = "";
-			if (shouldProcessMarkdown && (!onPage || generateIndividualMd)) md = htmlToMarkdown(html, withMinimalPreset({ origin: pageOrigin }));
+			if (shouldProcessMarkdown) md = htmlToMarkdown(html, withMinimalPreset({ origin: pageOrigin }));
 			let filePath;
 			if (shouldProcessMarkdown && generateIndividualMd) {
 				const urlObj = new URL(request.loadedUrl);
-				const safeSegments = (urlObj.pathname === "/" ? "/index" : urlObj.pathname).replace(/\/$/, "").split("/").filter((seg) => seg.length > 0).map((seg) => seg.replace(/[^\w\-]/g, "-"));
+				const safeSegments = (urlObj.pathname === "/" ? "/index" : urlObj.pathname).replace(URL_TRAILING_SLASH_RE, "").split("/").filter((seg) => seg.length > 0).map((seg) => seg.replace(URL_PATH_UNSAFE_CHARS_RE, "-"));
 				filePath = join(outputDir, normalize(`${safeSegments.length > 0 ? safeSegments.join("/") : "index"}.md`));
 				const fileDir = dirname(filePath);
 				if (fileDir && !existsSync(fileDir)) mkdirSync(fileDir, { recursive: true });
 				await writeFile(filePath, md, "utf-8");
 			}
-			const isHomePage = request.loadedUrl.replace(/\/$/, "") === homePageUrl.replace(/\/$/, "");
+			const isHomePage = request.loadedUrl.replace(URL_TRAILING_SLASH_RE, "") === homePageUrl.replace(URL_TRAILING_SLASH_RE, "");
 			if (shouldProcessMarkdown || isHomePage) {
 				const result = {
 					url: request.loadedUrl,
@@ -543,10 +549,10 @@ async function crawlAndGenerate(options, onProgress) {
 		onProgress?.(progress);
 		const successfulResults = results.filter((r) => r.success);
 		const firstUrl = new URL(withHttps(urls[0]));
-		const origin$1 = firstUrl.origin;
+		const origin = firstUrl.origin;
 		const homePageResult = successfulResults.find((r) => {
 			const resultUrl = new URL(withHttps(r.url));
-			return resultUrl.href === origin$1 || resultUrl.href === `${origin$1}/`;
+			return resultUrl.href === origin || resultUrl.href === `${origin}/`;
 		});
 		const siteName = siteNameOverride || homePageResult?.metadata?.title || homePageResult?.title || firstUrl.hostname;
 		const description = descriptionOverride || homePageResult?.metadata?.description || successfulResults[0]?.metadata?.description;
@@ -555,7 +561,7 @@ async function crawlAndGenerate(options, onProgress) {
 			onProgress?.(progress);
 			const contentResults = successfulResults.filter((result) => {
 				if (!result.content) return false;
-				return result.content.trim().replace(/^---\s*\n(?:.*\n)*?---\s*/, "").trim().length > 10;
+				return result.content.trim().replace(FRONTMATTER_BLOCK_RE, "").trim().length > 10;
 			});
 			const seenUrls = /* @__PURE__ */ new Set();
 			const llmsResult = await generateLlmsTxtArtifacts({
@@ -572,7 +578,7 @@ async function crawlAndGenerate(options, onProgress) {
 				})),
 				siteName,
 				description,
-				origin: origin$1 || firstUrl.origin,
+				origin: origin || firstUrl.origin,
 				generateFull: generateLlmsFullTxt,
 				outputDir
 			});
@@ -593,6 +599,5 @@ async function crawlAndGenerate(options, onProgress) {
 	await purgeDefaultStorages();
 	return results;
 }
-
 //#endregion
-export { parseUrlPattern as n, validateGlobPattern as r, crawlAndGenerate as t };
+export { parseUrlPattern as n, validateGlobPattern as r, crawlAndGenerate as t };

package/dist/cli.mjs

@@ -6,7 +6,6 @@ import { dirname, join, resolve } from "pathe";
 import { withHttps } from "ufo";
 import { fileURLToPath } from "node:url";
 import { addDependency } from "nypm";
-
 //#region src/playwright-utils.ts
 async function checkPlaywrightInstallation() {
 	try {
@@ -61,7 +60,6 @@ async function isUseChromeSupported() {
 	} catch {}
 	return false;
 }
-
 //#endregion
 //#region src/cli.ts
 const packageJsonPath = join(dirname(fileURLToPath(import.meta.url)), "..", "package.json");
@@ -106,8 +104,8 @@ async function interactiveCrawl() {
 		placeholder: "e.g. docs.example.com, site.com/docs/**",
 		validate: (value) => {
 			if (!value) return "Please enter at least one URL";
-			const urls$1 = value.split(",").map((url) => url.trim());
-			for (const url of urls$1) {
+			const urls = value.split(",").map((url) => url.trim());
+			for (const url of urls) {
 				const globError = validateGlobPattern(url);
 				if (globError) return globError;
 				try {
@@ -210,7 +208,7 @@ async function interactiveCrawl() {
 		inferredOrigin && `Origin: ${inferredOrigin}`
 	].filter(Boolean);
 	p.note(summary.join("\n"), "Crawl Configuration");
-	if (advancedOptions.skipSitemap && globPatterns.some((p$1) => p$1.isGlob)) p.log.warn("Warning: Using --skip-sitemap with glob URLs may not discover all matching pages.");
+	if (advancedOptions.skipSitemap && globPatterns.some((p) => p.isGlob)) p.log.warn("Warning: Using --skip-sitemap with glob URLs may not discover all matching pages.");
 	return {
 		urls,
 		outputDir: resolve(outputDir),
@@ -493,6 +491,5 @@ main().catch((error) => {
 	p.log.error(`Unexpected error: ${error}`);
 	process.exit(1);
 });
-
 //#endregion
-export {  };
+export {};

package/dist/index.mjs

@@ -1,8 +1,8 @@
 import { t as crawlAndGenerate } from "./_chunks/crawl.mjs";
 import { writeFile } from "node:fs/promises";
 import { basename, sep } from "pathe";
-
 //#region src/llms-txt.ts
+const ANCHOR_UNSAFE_CHARS_RE = /[^a-z0-9]/g;
 async function generateLlmsTxt(options) {
 	const { siteName, description, results, outputPath } = options;
 	let content = `# ${siteName}\n\n`;
@@ -22,8 +22,8 @@ async function generateLlmsTxt(options) {
 				const linkPath = (mdIndex !== -1 ? result.filePath.substring(mdIndex + mdSeparator.length) : basename(result.filePath)).split(sep).join("/");
 				content += `- [${title}](md/${linkPath}): ${result.url}\n`;
 			} else {
-				const description$1 = result.metadata?.description ? result.metadata.description.split("\n")[0].substring(0, 100) + (result.metadata.description.length > 100 ? "..." : "") : "";
-				content += `- [${title}](${result.url})${description$1 ? `: ${description$1}` : ""}\n`;
+				const description = result.metadata?.description ? result.metadata.description.split("\n")[0].substring(0, 100) + (result.metadata.description.length > 100 ? "..." : "") : "";
+				content += `- [${title}](${result.url})${description ? `: ${description}` : ""}\n`;
 			}
 		}
 	}
@@ -42,7 +42,7 @@ async function generateLlmsFullTxt(options) {
 			} catch {
 				title = result.title || result.url;
 			}
-			const anchor = title.toLowerCase().replace(/[^a-z0-9]/g, "-");
+			const anchor = title.toLowerCase().replace(ANCHOR_UNSAFE_CHARS_RE, "-");
 			content += `- [${title}](#${anchor})\n`;
 		}
 		content += `\n---\n\n`;
@@ -60,6 +60,5 @@ async function generateLlmsFullTxt(options) {
 	}
 	await writeFile(outputPath, content, "utf-8");
 }
-
 //#endregion
-export { crawlAndGenerate, generateLlmsFullTxt, generateLlmsTxt };
+export { crawlAndGenerate, generateLlmsFullTxt, generateLlmsTxt };

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@mdream/crawl",
   "type": "module",
-  "version": "0.15.3",
+  "version": "0.17.0",
   "description": "Mdream Crawl generates comprehensive llms.txt artifacts from a single URL, using mdream to convert HTML to Markdown.",
   "author": {
     "name": "Harlan Wilton",
@@ -50,13 +50,13 @@
     }
   },
   "dependencies": {
-    "@clack/prompts": "^0.11.0",
-    "crawlee": "^3.15.3",
-    "nypm": "^0.6.2",
+    "@clack/prompts": "^1.1.0",
+    "crawlee": "^3.16.0",
+    "nypm": "^0.6.5",
     "pathe": "^2.0.3",
     "picomatch": "^4.0.3",
-    "ufo": "^1.6.1",
-    "mdream": "0.15.3"
+    "ufo": "^1.6.3",
+    "mdream": "0.17.0"
   },
   "devDependencies": {
     "@types/picomatch": "^4.0.2"