clean-web-scraper 3.3.1 → 3.3.3

package/example-usage.js

@@ -76,14 +76,14 @@ async function bdsmovement ()
 
 void async function main ()
 {
-	// const khameneiIrFreePalestineTagScraper = await khameneiIrFreePalestineTag();
-	// const decolonizepalestineScraper = await decolonizepalestine();
-	const bdsmovementScraper = await bdsmovement();
-	// await WebScraper.combineResults( "./dataset/combined", [
-	// 	khameneiIrFreePalestineTagScraper,
-	// 	decolonizepalestineScraper,
-	// 	bdsmovementScraper
-	// ] );
+	const khameneiIrFreePalestineTagScraper = await khameneiIrFreePalestineTag();
+	const decolonizepalestineScraper = await decolonizepalestine();
+	// const bdsmovementScraper = await bdsmovement();
+	await WebScraper.combineResults( "./dataset/combined", [
+		khameneiIrFreePalestineTagScraper,
+		decolonizepalestineScraper,
+		// bdsmovementScraper
+	] );
 
 	// 4
 	// https://electronicintifada.net/

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "clean-web-scraper",
-  "version": "3.3.1",
+  "version": "3.3.3",
   "main": "main.js",
   "scripts": {
     "start": "node main.js",

package/src/WebScraper.js

@@ -19,7 +19,11 @@ class WebScraper
 		csvOutputPath,
 		includeMetadata = false,
 		metadataFields = [], // ['title', 'description', 'author', 'lastModified', etc.]
-		userAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:134.0) Gecko/20100101 Firefox/134.0"
+		headers = {
+			"user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:134.0) Gecko/20100101 Firefox/134.0",
+			"Cache-Control": "private",
+			"Accept": "application/xml,application/xhtml+xml,text/html;q=0.9, text/plain;q=0.8,image/png,*/*;q=0.5"
+		}
 	})
 	{
 		this.baseURL = baseURL;
@@ -31,7 +35,7 @@ class WebScraper
 		this.csvOutputPath = csvOutputPath || path.join( this.scrapResultPath, "train.csv" );
 		this.jsonlOutputPathWithMeta = jsonlOutputPath.replace( ".jsonl", "_with_metadata.jsonl" );
 		this.csvOutputPathWithMeta = csvOutputPath.replace( ".csv", "_with_metadata.csv" );
-		this.userAgent = userAgent;
+		this.headers = headers;
 		this.includeMetadata = includeMetadata;
 	   this.metadataFields = new Set( metadataFields );
 		this.visited = new Set();
@@ -60,9 +64,7 @@ class WebScraper
 		try
 		{
 			const { data, headers } = await axios.get( url, {
-				headers: {
-					"user-agent": this.userAgent
-				}
+				headers: this.headers,
 			});
 			const dom = new JSDOM( data, { url });
 			const { document } = dom.window;
@@ -74,9 +76,17 @@ class WebScraper
 
 				if ( article )
 				{
-					const metadata = this.metadataextractor( url, document, headers );
-					metadata.depth = depth;
-					this.saveArticle( url, article.textContent, metadata );
+					if ( this.isValidContent( article.textContent ) )
+					{
+
+						const metadata = this.metadataextractor( url, document, headers );
+						metadata.depth = depth;
+						this.saveArticle( url, article.textContent, metadata );
+					}
+					else
+					{
+						console.error( `Invalid content found at ${url}` );
+					}
 				}
 				else
 				{
@@ -373,12 +383,55 @@ class WebScraper
 		if ( fs.existsSync( path.join( __dirname, this.scrapResultPath ) ) )
 		{
 			fs.rmSync( path.join( __dirname, this.scrapResultPath ), { recursive: true, force: true });
+		}
+		if ( fs.existsSync( path.join( __dirname, this.textOutputPath ) ) )
+		{
 			fs.rmSync( path.join( __dirname, this.textOutputPath ), { recursive: true, force: true });
 		}
+		if ( fs.existsSync( path.join( __dirname, this.csvOutputPath ) ) )
+		{
+			fs.rmSync( path.join( __dirname, this.csvOutputPath ), { recursive: true, force: true });
+		}
+		if ( fs.existsSync( path.join( __dirname, this.csvOutputPathWithMeta ) ) )
+		{
+			fs.rmSync( path.join( __dirname, this.csvOutputPathWithMeta ), { recursive: true, force: true });
+		}
+		if ( fs.existsSync( path.join( __dirname, this.jsonlOutputPath ) ) )
+		{
+			fs.rmSync( path.join( __dirname, this.jsonlOutputPath ), { recursive: true, force: true });
+		}
+		if ( fs.existsSync( path.join( __dirname, this.jsonlOutputPathWithMeta ) ) )
+		{
+			fs.rmSync( path.join( __dirname, this.jsonlOutputPathWithMeta ), { recursive: true, force: true });
+		}
 		fs.mkdirSync( path.join( __dirname, this.scrapResultPath ), { recursive: true });
 		fs.mkdirSync( path.join( __dirname, this.textOutputPath ), { recursive: true });
 	}
 
+	isValidContent ( content )
+	{
+		// Remove whitespace and newlines for checking
+		const cleanContent = content.replace( /\s+/g, " " ).trim().toLowerCase();
+
+		// List of phrases that indicate invalid content
+		const invalidPhrases = [
+			"verifying that you are not a robot",
+			"checking if the site connection is secure",
+			"please wait while we verify",
+			"please enable javascript",
+			"access denied",
+			"captcha verification"
+		];
+
+		const hasInvalidPhrases = invalidPhrases.some( phrase => { return cleanContent.includes( phrase ) });
+		// Check content length
+		if ( cleanContent.length < 100 && hasInvalidPhrases )
+		{
+			return false;
+		}
+		return true;
+	}
+
 	static sleep ( ms )
 	{
 		return new Promise( resolve => { return setTimeout( resolve, ms ) });
@@ -419,12 +472,18 @@ class WebScraper
 		for ( const website of websites )
 		{
 			const jsonlContent = fs.readFileSync( path.join( __dirname, website.jsonlOutputPath ), "utf-8" );
-			jsonlOutput.write( jsonlContent );
+			if ( jsonlContent )
+			{
+				jsonlOutput.write( jsonlContent );
+			}
 
 			if ( website.includeMetadata )
 			{
 				const jsonlMetaContent = fs.readFileSync( path.join( __dirname, website.jsonlOutputPathWithMeta ), "utf-8" );
-				jsonlMetaOutput.write( jsonlMetaContent );
+				if ( jsonlMetaContent )
+				{
+					jsonlMetaOutput.write( jsonlMetaContent );
+				}
 			}
 		}
 
@@ -451,7 +510,10 @@ class WebScraper
 			.split( "\n" )
 			.slice( 1 )
 			.filter( line => { return line.trim() });
-			csvOutput.write( `${csvContent.join( "\n" )}\n` );
+			if ( csvContent.length > 0 )
+			{
+				csvOutput.write( `${csvContent.join( "\n" )}\n` );
+			}
 
 			if ( website.includeMetadata )
 			{
@@ -459,7 +521,10 @@ class WebScraper
 				.split( "\n" )
 				.slice( 1 )
 				.filter( line => { return line.trim() });
-				csvMetaOutput.write( `${csvMetaContent.join( "\n" )}\n` );
+				if ( csvMetaContent.length > 0 )
+				{
+					csvMetaOutput.write( `${csvMetaContent.join( "\n" )}\n` );
+				}
 			}
 		}
 
@@ -500,7 +565,6 @@ class WebScraper
 			}
 		}
 	}
-
 }
 
 module.exports = WebScraper;