clean-web-scraper 3.7.4 → 3.7.6

package/main.js

@@ -130,9 +130,8 @@ class WebScraper
 
 	async fetchPage ( url, depth )
 	{
-		if ( this.allProcessedContent.length >= this.maxArticles || depth > this.maxDepth )
+		if ( this.hasReachedMax( depth ) )
 		{
-			console.error( `Reached maximum: ${this.allProcessedContent.length}, ${this.maxDepth}` );
 			return;
 		}
 		if ( this.visited.has( url ) )
@@ -181,6 +180,10 @@ class WebScraper
 
 			for ( let i = 0; i < unvisitedLinks.length; i += this.concurrencyLimit )
 			{
+				if ( this.hasReachedMax( depth ) )
+				{
+					return;
+				}
 				await WebScraper.sleep( 5000 );
 				const batch = unvisitedLinks.slice( i, i + this.concurrencyLimit );
 				const results = await Promise.allSettled( batch.map( link => { return this.fetchPage( link, depth + 1 ) }) );
@@ -205,8 +208,8 @@ class WebScraper
 		try
 		{
 			const response = await this.retryAxiosRequest( url )
-			const contentType = response.headers["content-type"] || "";
-			if ( !contentType.startsWith( "text" ) )
+			const contentType = response?.headers["content-type"] || "";
+			if ( !contentType?.startsWith( "text" ) )
 			{
 				console.log( `Skipping non-HTML content for ${url}: Content-Type is ${contentType}` );
 				response.data.destroy();
@@ -259,6 +262,16 @@ class WebScraper
 		}
 	}
 
+	hasReachedMax ( depth )
+	{
+		if ( this.allProcessedContent.length >= this.maxArticles || depth > this.maxDepth )
+		{
+			console.error( `Reached maximum: ${this.allProcessedContent.length}, ${this.maxDepth} , ${depth}` );
+			return true;
+		}
+		return false;
+	}
+
 	async navigateToPage ( url )
 	{
 		let pages = await this.puppeteerBrowser.pages();
@@ -559,15 +572,20 @@ class WebScraper
 		{
 			try
 			{
+				if ( this.hasReachedMax( depth ) )
+				{
+					throw new Error( "Max reached" );
+				}
 				return await axios.get( url, options );
 			}
 			catch ( error )
 			{
-				if ( attempt === this.maxRetries ) throw error;
+				if ( attempt >= this.maxRetries ) throw error;
 				await WebScraper.sleep( 40000 * attempt );
 				console.error( `Retrying request to ${url} (Attempt ${attempt + 1}/${this.maxRetries})`, error.message, error.code );
 			}
 		}
+		throw new Error( "Max retries reached" );
 	}
 
 	configurePuppeteer ( )

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "clean-web-scraper",
-  "version": "3.7.4",
+  "version": "3.7.6",
   "main": "main.js",
   "scripts": {
     "start": "node main.js",