clean-web-scraper 3.3.6 → 3.3.8

package/example-usage.js

@@ -100,7 +100,8 @@ async function electronicintifada ()
 			"https://electronicintifada.net/tags/",
 			"https://electronicintifada.net/blog",
 			"https://electronicintifada.net/people",
-			"https://electronicintifada.net/location"
+			"https://electronicintifada.net/location",
+			"https://electronicintifada.net/file"
 		],
 		exactExcludeList: [
 			"https://electronicintifada.net",
@@ -133,9 +134,6 @@ void async function main ()
 		electronicintifadaScraper
 	] );
 
-	// 4
-	// https://electronicintifada.net/
-
 	// 5
 	// https://www.palestineremembered.com/ZionistFAQ.html
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "clean-web-scraper",
-  "version": "3.3.6",
+  "version": "3.3.8",
   "main": "main.js",
   "scripts": {
     "start": "node main.js",

package/src/WebScraper.js

@@ -20,12 +20,14 @@ class WebScraper
 		textOutputPath,
 		csvOutputPath,
 		includeMetadata = false,
-		metadataFields = [], // ['title', 'description', 'author', 'lastModified', etc.]
+		metadataFields = [], // ['title', 'description', 'author', etc.]
 		headers,
 		usePuppeteer,
 		puppeteerProxy, // e.g. http://127.0.0.1:2080
 		puppeteerExecutablePath,
-		puppeteerRealProxy
+		puppeteerRealProxy,
+		filterFileTypes = true,
+		excludedFileTypes = [".mp3", ".mp4", ".wav", ".avi", ".mov", ".pdf", ".zip", ".rar"]
 	})
 	{
 		this.baseURL = baseURL;
@@ -44,6 +46,8 @@ class WebScraper
 		this.excludeList = this.normalizeExcludeList( excludeList );
 		this.exactExcludeList = this.normalizeExcludeList( exactExcludeList );
 		this.allProcessedContent = [];
+		this.filterFileTypes = filterFileTypes;
+		this.excludedFileTypes = excludedFileTypes;
 		this.usePuppeteer = usePuppeteer || false;
 		this.puppeteerOptions = {
 			headless: false,
@@ -120,7 +124,7 @@ class WebScraper
 			const dom = new JSDOM( data, { url });
 			const { document } = dom.window;
 
-			if ( !this.isExcluded( url ) )
+			if ( !this.isExcluded( url ) && this.isValidFileType( url ) )
 			{
 				const reader = new Readability( document, { charThreshold: 500, nbTopCandidates: 20 });
 				const article = reader.parse();
@@ -548,6 +552,13 @@ class WebScraper
 		fs.mkdirSync( path.join( __dirname, this.textOutputPath ), { recursive: true });
 	}
 
+	isValidFileType ( url )
+	{
+		if ( !this.filterFileTypes ) return true;
+		const urlPath = new URL( url ).pathname.toLowerCase();
+		return !this.excludedFileTypes.some( ext => { return urlPath.endsWith( ext ) });
+	}
+
 	isValidContent ( content )
 	{
 		// Remove whitespace and newlines for checking