clean-web-scraper 3.5.3 → 3.5.5

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "clean-web-scraper",
-  "version": "3.5.3",
+  "version": "3.5.5",
   "main": "main.js",
   "scripts": {
     "start": "node main.js",

package/src/WebScraper.js

@@ -73,6 +73,15 @@ class WebScraper
 		// Network configuration
 		this.axiosHeaders = axiosHeaders;
 		this.axiosProxy = axiosProxy;
+		this.axiosOptions = {};
+		if ( this.axiosHeaders )
+		{
+			axiosOptions.headers = this.axiosHeaders;
+		}
+		if ( this.axiosProxy )
+		{
+			axiosOptions.proxy = this.axiosProxy;
+		}
 
 		// Content storage
 		this.allProcessedContent = [];
@@ -128,17 +137,13 @@ class WebScraper
 			return;
 		}
 		this.visited.add( url );
-		if ( !this.isValidFileType( url ) )
-		{
-			return;
-		}
-		if ( !this.isValidDomain( url ) )
+		if ( !this.isValidFileType( url ) || !this.isValidDomain( url ) )
 		{
 			return;
 		}
 		try
 		{
-			const data = await this.caller( url );
+			const data = await this.fetchContent( url );
 			if ( !data ) return;
 			const dom = new JSDOM( data, { url });
 			const { document } = dom.window;
@@ -150,9 +155,9 @@ class WebScraper
 
 				if ( article )
 				{
-					if ( this.isValidContent( article.textContent ) )
+					if ( this.hasValidPageContent( article.textContent ) )
 					{
-						const metadata = this.metadataextractor( url, document );
+						const metadata = this.extractMetadata( url, document );
 						metadata.depth = depth;
 						this.saveArticle( url, article.textContent, metadata );
 					}
@@ -182,35 +187,23 @@ class WebScraper
 		}
 	}
 
-	async caller ( url )
+	async fetchContent ( url )
 	{
 		try
 		{
-			let axiosOptions = {};
-			if ( this.axiosHeaders )
-			{
-				axiosOptions.headers = this.axiosHeaders;
-			}
-			if ( this.axiosProxy )
-			{
-				axiosOptions.proxy = this.axiosProxy;
-			}
-
-			// Step 1: Make a GET request with a small timeout and limited data download
 			const response = await axios.get( url, {
-				...axiosOptions,
 				responseType: "stream",
 				maxRedirects: 5,
-				timeout: 70000
+				timeout: 70000,
+				...axiosOptions,
 			});
 
-			// Step 2: Check the Content-Type header from the response
 			const contentType = response.headers["content-type"] || "";
 			if ( !contentType.startsWith( "text" ) )
 			{
 				console.log( `Skipping non-HTML content for ${url}: Content-Type is ${contentType}` );
-				response.data.destroy(); // Destroy the stream to stop downloading further data
-				return null; // Skip further processing for non-HTML content
+				response.data.destroy();
+				return null;
 			}
 
 			// Step 3: If Content-Type is HTML, read the full response
@@ -242,8 +235,8 @@ class WebScraper
 					for ( let index = 0; index < 10; index++ )
 					{
 						console.log( `Please solve the CAPTCHA on the opened browser window for ${url}` );
-						result = await this.goToUrl( url ) ;
-						if ( this.isValidContent( result.htmlContent ) )
+						result = await this.navigateToPage( url ) ;
+						if ( this.hasValidPageContent( result.htmlContent ) )
 						{
 							break
 						}
@@ -261,7 +254,7 @@ class WebScraper
 		}
 	}
 
-	async goToUrl ( url )
+	async navigateToPage ( url )
 	{
 		let pages = await this.puppeteerBrowser.pages();
 		let page = pages[0];
@@ -530,7 +523,7 @@ class WebScraper
 		return filteredMetadata;
 	}
 
-	metadataextractor ( url, document )
+	extractMetadata ( url, document )
 	{
 		return {
 			url,
@@ -634,7 +627,7 @@ class WebScraper
 		}
 	}
 
-	isValidContent ( content )
+	hasValidPageContent ( content )
 	{
 		// Remove whitespace and newlines for checking
 		const cleanContent = content.replace( /\s+/g, " " ).trim().toLowerCase();