@mdream/crawl 0.7.1 → 0.7.2

package/dist/_chunks/{crawl-B5MaCj6O.mjs → crawl-DEysrw0h.mjs}

@@ -1,6 +1,6 @@
 import { existsSync, mkdirSync } from "node:fs";
 import { writeFile } from "node:fs/promises";
-import { HttpCrawler, Sitemap, purgeDefaultStorages } from "crawlee";
+import { HttpCrawler, PlaywrightCrawler, Sitemap, purgeDefaultStorages } from "crawlee";
 import { generateLlmsTxtArtifacts, htmlToMarkdown } from "mdream";
 import { withMinimalPreset } from "mdream/preset/minimal";
 import { dirname, join, normalize, resolve } from "pathe";
@@ -321,8 +321,7 @@ async function crawlAndGenerate(options, onProgress) {
 			const startTime = Date.now();
 			progress.crawling.currentUrl = request.loadedUrl;
 			onProgress?.(progress);
-			const baseUrl = new URL(startingUrls[0]).origin;
-			const homePageUrl = baseUrl;
+			const homePageUrl = new URL(startingUrls[0]).origin;
 			let html;
 			let title;
 			if (crawlerType === "playwright") {
@@ -385,14 +384,15 @@ async function crawlAndGenerate(options, onProgress) {
 	let crawler;
 	const crawlerOptions = {
 		requestHandler: createRequestHandler(driver),
+		errorHandler: async ({ request, response }, error) => {
+			if (response?.statusCode && response?.statusCode >= 400) request.noRetry = true;
+		},
 		maxRequestsPerCrawl,
 		respectRobotsTxtFile: true
 	};
-	if (crawlDelay) crawlerOptions.requestHandlerTimeoutMillis = crawlDelay * 1e3;
-	if (driver === "playwright") {
-		const { PlaywrightCrawler: PlaywrightCrawlerClass } = await import("crawlee");
-		crawler = new PlaywrightCrawlerClass(crawlerOptions);
-	} else crawler = new HttpCrawler(crawlerOptions);
+	if (crawlDelay) crawlerOptions.requestHandlerTimeoutSecs = crawlDelay;
+	if (driver === "playwright") crawler = new PlaywrightCrawler(crawlerOptions);
+	else crawler = new HttpCrawler(crawlerOptions);
 	const initialRequests = startingUrls.map((url) => ({
 		url,
 		userData: { depth: 0 }

package/dist/cli.mjs

@@ -1,4 +1,4 @@
-import { crawlAndGenerate, parseUrlPattern, validateGlobPattern, withHttps } from "./_chunks/crawl-B5MaCj6O.mjs";
+import { crawlAndGenerate, parseUrlPattern, validateGlobPattern, withHttps } from "./_chunks/crawl-DEysrw0h.mjs";
 import { readFileSync } from "node:fs";
 import { dirname, join, resolve } from "pathe";
 import { fileURLToPath } from "node:url";

package/dist/index.mjs

@@ -1,4 +1,4 @@
-import { crawlAndGenerate } from "./_chunks/crawl-B5MaCj6O.mjs";
+import { crawlAndGenerate } from "./_chunks/crawl-DEysrw0h.mjs";
 import { writeFile } from "node:fs/promises";
 import { basename, sep } from "pathe";
 

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@mdream/crawl",
   "type": "module",
-  "version": "0.7.1",
+  "version": "0.7.2",
   "description": "Mdream Crawl generates comprehensive llms.txt artifacts from a single URL, using mdream to convert HTML to Markdown.",
   "author": {
     "name": "Harlan Wilton",
@@ -46,11 +46,11 @@
   },
   "dependencies": {
     "@clack/prompts": "^0.11.0",
-    "crawlee": "^3.13.9",
+    "crawlee": "^3.13.10",
     "nypm": "^0.6.0",
     "pathe": "^2.0.3",
     "picomatch": "^4.0.2",
-    "mdream": "0.7.1"
+    "mdream": "0.7.2"
   },
   "devDependencies": {
     "@types/picomatch": "^4.0.0"