clean-web-scraper 3.3.0 → 3.3.2

package/example-usage.js

@@ -3,15 +3,12 @@ const WebScraper = require( "./src/WebScraper" );
 
 async function khameneiIrFreePalestineTag ()
 {
-	// 1
 	// https://english.khamenei.ir/Opinions/FreePalestine
 	// https://english.khamenei.ir/page/search.xhtml?topicid=0&period=0&q=FreePalestine&pageSize=100#
 	const scraper = new WebScraper({
 		baseURL: "https://english.khamenei.ir/news",
 		startURL: "https://english.khamenei.ir/page/search.xhtml?topicid=0&period=0&q=FreePalestine&pageSize=100#",
 		maxDepth: 1,
-		excludeList: [
-		],
 		exactExcludeList: [
 			"https://english.khamenei.ir/page/search.xhtml?topicid=0&period=0&q=FreePalestine&pageSize=100#"
 		],
@@ -20,7 +17,7 @@ async function khameneiIrFreePalestineTag ()
 		textOutputPath: "./dataset/khamenei-ir-free-palestine-tag/texts",
 		csvOutputPath: "./dataset/khamenei-ir-free-palestine-tag/train.csv",
 		includeMetadata: true,
-		metadataFields: ["title", "description", "author", "lastModified", "language"]
+		metadataFields: ["title", "description", "author"]
 	});
 	await scraper.start();
 	return scraper;
@@ -28,7 +25,6 @@ async function khameneiIrFreePalestineTag ()
 
 async function decolonizepalestine ()
 {
-	// 2
 	// https://decolonizepalestine.com
 	const scraper = new WebScraper({
 		baseURL: "https://decolonizepalestine.com",
@@ -48,23 +44,46 @@ async function decolonizepalestine ()
 		textOutputPath: "./dataset/decolonizepalestine/texts",
 		csvOutputPath: "./dataset/decolonizepalestine/train.csv",
 		includeMetadata: true,
-		metadataFields: ["title", "description", "author", "lastModified", "language"]
+		metadataFields: ["title", "description", "author"]
 	});
 	await scraper.start();
 	return scraper;
 }
 
-void async function main ()
+async function bdsmovement ()
 {
-	const khameneiIrFreePalestineTagScraper = await khameneiIrFreePalestineTag();
-	const decolonizepalestineScraper = await decolonizepalestine();
-	await WebScraper.combineResults( "./dataset/combined", [
-		khameneiIrFreePalestineTagScraper,
-		decolonizepalestineScraper
-	] );
+	// https://bdsmovement.org
+	const scraper = new WebScraper({
+		baseURL: "https://bdsmovement.org",
+		excludeList: [
+			"https://bdsmovement.net/press-area",
+			"https://bdsmovement.net/privacy-policy",
+			"https://bdsmovement.net/get-involved/join-a-bds-campaign",
+			"https://bdsmovement.net/donate_",
+			"https://bdsmovement.net/user",
+			"https://bdsmovement.net/admin"
+		],
+		scrapResultPath: "./dataset/bdsmovement/website",
+		jsonlOutputPath: "./dataset/bdsmovement/train.jsonl",
+		textOutputPath: "./dataset/bdsmovement/texts",
+		csvOutputPath: "./dataset/bdsmovement/train.csv",
+		includeMetadata: true,
+		metadataFields: ["title", "description", "author"]
+	});
+	await scraper.start();
+	return scraper;
+}
 
-	// 3
-	// https://bdsmovement.net
+void async function main ()
+{
+	// const khameneiIrFreePalestineTagScraper = await khameneiIrFreePalestineTag();
+	// const decolonizepalestineScraper = await decolonizepalestine();
+	const bdsmovementScraper = await bdsmovement();
+	// await WebScraper.combineResults( "./dataset/combined", [
+	// 	khameneiIrFreePalestineTagScraper,
+	// 	decolonizepalestineScraper,
+	// 	bdsmovementScraper
+	// ] );
 
 	// 4
 	// https://electronicintifada.net/

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "clean-web-scraper",
-  "version": "3.3.0",
+  "version": "3.3.2",
   "main": "main.js",
   "scripts": {
     "start": "node main.js",

package/src/WebScraper.js

@@ -18,7 +18,12 @@ class WebScraper
 		textOutputPath,
 		csvOutputPath,
 		includeMetadata = false,
-		metadataFields = [] // ['title', 'description', 'author', 'lastModified', etc.]
+		metadataFields = [], // ['title', 'description', 'author', 'lastModified', etc.]
+		headers = {
+			"user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:134.0) Gecko/20100101 Firefox/134.0",
+			"Cache-Control": "private",
+			"Accept": "application/xml,application/xhtml+xml,text/html;q=0.9, text/plain;q=0.8,image/png,*/*;q=0.5"
+		}
 	})
 	{
 		this.baseURL = baseURL;
@@ -30,10 +35,11 @@ class WebScraper
 		this.csvOutputPath = csvOutputPath || path.join( this.scrapResultPath, "train.csv" );
 		this.jsonlOutputPathWithMeta = jsonlOutputPath.replace( ".jsonl", "_with_metadata.jsonl" );
 		this.csvOutputPathWithMeta = csvOutputPath.replace( ".csv", "_with_metadata.csv" );
+		this.headers = headers;
 		this.includeMetadata = includeMetadata;
 	   this.metadataFields = new Set( metadataFields );
 		this.visited = new Set();
-		this.excludeList = new Set( excludeList );
+		this.excludeList = this.normalizeExcludeList( excludeList );
 		this.exactExcludeList = this.normalizeExcludeList( exactExcludeList );
 		this.allProcessedContent = [];
 	}
@@ -57,7 +63,9 @@ class WebScraper
 		this.visited.add( url );
 		try
 		{
-			const { data, headers } = await axios.get( url );
+			const { data, headers } = await axios.get( url, {
+				headers: this.headers,
+			});
 			const dom = new JSDOM( data, { url });
 			const { document } = dom.window;
 
@@ -68,9 +76,17 @@ class WebScraper
 
 				if ( article )
 				{
-					const metadata = this.metadataextractor( url, document, headers );
-					metadata.depth = depth;
-					this.saveArticle( url, article.textContent, metadata );
+					if ( this.isValidContent( article.textContent ) )
+					{
+
+						const metadata = this.metadataextractor( url, document, headers );
+						metadata.depth = depth;
+						this.saveArticle( url, article.textContent, metadata );
+					}
+					else
+					{
+						console.error( `Invalid content found at ${url}` );
+					}
 				}
 				else
 				{
@@ -334,7 +350,7 @@ class WebScraper
 		};
 	}
 
-	normalizeExcludeList ( list )
+	normalizeExcludeList ( list = [] )
 	{
 		const normalizedSet = new Set();
 		for ( let i = 0; i < list.length; i++ )
@@ -367,8 +383,27 @@ class WebScraper
 		if ( fs.existsSync( path.join( __dirname, this.scrapResultPath ) ) )
 		{
 			fs.rmSync( path.join( __dirname, this.scrapResultPath ), { recursive: true, force: true });
+		}
+		if ( fs.existsSync( path.join( __dirname, this.textOutputPath ) ) )
+		{
 			fs.rmSync( path.join( __dirname, this.textOutputPath ), { recursive: true, force: true });
 		}
+		if ( fs.existsSync( path.join( __dirname, this.csvOutputPath ) ) )
+		{
+			fs.rmSync( path.join( __dirname, this.csvOutputPath ), { recursive: true, force: true });
+		}
+		if ( fs.existsSync( path.join( __dirname, this.csvOutputPathWithMeta ) ) )
+		{
+			fs.rmSync( path.join( __dirname, this.csvOutputPathWithMeta ), { recursive: true, force: true });
+		}
+		if ( fs.existsSync( path.join( __dirname, this.jsonlOutputPath ) ) )
+		{
+			fs.rmSync( path.join( __dirname, this.jsonlOutputPath ), { recursive: true, force: true });
+		}
+		if ( fs.existsSync( path.join( __dirname, this.jsonlOutputPathWithMeta ) ) )
+		{
+			fs.rmSync( path.join( __dirname, this.jsonlOutputPathWithMeta ), { recursive: true, force: true });
+		}
 		fs.mkdirSync( path.join( __dirname, this.scrapResultPath ), { recursive: true });
 		fs.mkdirSync( path.join( __dirname, this.textOutputPath ), { recursive: true });
 	}
@@ -461,6 +496,30 @@ class WebScraper
 		csvMetaOutput.end();
 	}
 
+	isValidContent ( content )
+	{
+		// Remove whitespace and newlines for checking
+		const cleanContent = content.replace( /\s+/g, " " ).trim().toLowerCase();
+
+		// List of phrases that indicate invalid content
+		const invalidPhrases = [
+			"verifying that you are not a robot",
+			"checking if the site connection is secure",
+			"please wait while we verify",
+			"please enable javascript",
+			"access denied",
+			"captcha verification"
+		];
+
+		const hasInvalidPhrases = invalidPhrases.some( phrase => { return cleanContent.includes( phrase ) });
+		// Check content length
+		if ( cleanContent.length < 100 && hasInvalidPhrases )
+		{
+			return false;
+		}
+		return true;
+	}
+
 	static combineTextFiles ( fullOutputPath, websites )
 	{
 		let textFileCounter = 1;