clean-web-scraper 3.3.5 → 3.3.7

package/example-usage.js

@@ -83,7 +83,7 @@ async function bdsmovement ()
 			host: "socks5://127.0.0.1",
 			port: "2080",
 		},
-		usePuppeteer: true
+		// usePuppeteer: true
 	});
 	await scraper.start();
 	return scraper;
@@ -95,12 +95,19 @@ async function electronicintifada ()
 	const scraper = new WebScraper({
 		baseURL: "https://electronicintifada.net",
 		excludeList: [
-			"https://electronicintifada.net/press-area",
-			"https://electronicintifada.net/privacy-policy",
-			"https://electronicintifada.net/get-involved/join-a-bds-campaign",
-			"https://electronicintifada.net/donate_",
-			"https://electronicintifada.net/user",
-			"https://electronicintifada.net/admin"
+			"https://electronicintifada.net/updates",
+			"https://electronicintifada.net/taxonomy/term/",
+			"https://electronicintifada.net/tags/",
+			"https://electronicintifada.net/blog",
+			"https://electronicintifada.net/people",
+			"https://electronicintifada.net/location"
+		],
+		exactExcludeList: [
+			"https://electronicintifada.net",
+			"https://electronicintifada.net/blog",
+			"https://electronicintifada.net/news",
+			"https://electronicintifada.net/opinion",
+			"https://electronicintifada.net/review",
 		],
 		scrapResultPath: "./dataset/electronicintifada/website",
 		jsonlOutputPath: "./dataset/electronicintifada/train.jsonl",
@@ -118,10 +125,12 @@ void async function main ()
 	const khameneiIrFreePalestineTagScraper = await khameneiIrFreePalestineTag();
 	const decolonizepalestineScraper = await decolonizepalestine();
 	const bdsmovementScraper = await bdsmovement();
+	const electronicintifadaScraper = await electronicintifada();
 	await WebScraper.combineResults( "./dataset/combined", [
 		khameneiIrFreePalestineTagScraper,
 		decolonizepalestineScraper,
-		bdsmovementScraper
+		bdsmovementScraper,
+		electronicintifadaScraper
 	] );
 
 	// 4

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "clean-web-scraper",
-  "version": "3.3.5",
+  "version": "3.3.7",
   "main": "main.js",
   "scripts": {
     "start": "node main.js",

package/src/WebScraper.js

@@ -25,7 +25,9 @@ class WebScraper
 		usePuppeteer,
 		puppeteerProxy, // e.g. http://127.0.0.1:2080
 		puppeteerExecutablePath,
-		puppeteerRealProxy
+		puppeteerRealProxy,
+		filterFileTypes = true,
+		excludedFileTypes = [".mp3", ".mp4", ".wav", ".avi", ".mov", ".pdf", ".zip", ".rar"]
 	})
 	{
 		this.baseURL = baseURL;
@@ -44,6 +46,8 @@ class WebScraper
 		this.excludeList = this.normalizeExcludeList( excludeList );
 		this.exactExcludeList = this.normalizeExcludeList( exactExcludeList );
 		this.allProcessedContent = [];
+		this.filterFileTypes = filterFileTypes;
+		this.excludedFileTypes = excludedFileTypes;
 		this.usePuppeteer = usePuppeteer || false;
 		this.puppeteerOptions = {
 			headless: false,
@@ -120,7 +124,7 @@ class WebScraper
 			const dom = new JSDOM( data, { url });
 			const { document } = dom.window;
 
-			if ( !this.isExcluded( url ) )
+			if ( !this.isExcluded( url ) && this.isValidFileType( url ) )
 			{
 				const reader = new Readability( document, { charThreshold: 500, nbTopCandidates: 20 });
 				const article = reader.parse();
@@ -437,6 +441,23 @@ class WebScraper
 		// processed = processed.replace(/\[.*?\]/g, ''); // Removes all content within square brackets
 		// processed = processed.replace(/\(.*?\)/g, ''); // Removes all content within parentheses
 
+		// Remove specified words from the end of content, handling multiple occurrences
+		const wordsToTrim = ["Facebook", "Twitter", "Donate Now", "Instagram"];
+		let changed = true;
+
+		while ( changed )
+		{
+			changed = false;
+			for ( let i = 0; i < wordsToTrim.length; i++ )
+			{
+				const oldProcessed = processed;
+				processed = processed.replace( new RegExp( `\\s*${wordsToTrim[i]}\\s*$`, "g" ), "" ).trim();
+				if ( oldProcessed !== processed )
+				{
+					changed = true;
+				}
+			}
+		}
 		return processed;
 	}
 
@@ -531,6 +552,13 @@ class WebScraper
 		fs.mkdirSync( path.join( __dirname, this.textOutputPath ), { recursive: true });
 	}
 
+	isValidFileType ( url )
+	{
+		if ( !this.filterFileTypes ) return true;
+		const urlPath = new URL( url ).pathname.toLowerCase();
+		return !this.excludedFileTypes.some( ext => { return urlPath.endsWith( ext ) });
+	}
+
 	isValidContent ( content )
 	{
 		// Remove whitespace and newlines for checking