clean-web-scraper 3.4.0 → 3.5.0

package/README.md

@@ -89,7 +89,8 @@ await WebScraper.combineResults('./combined', [docsScraper, blogScraper]);
 ```
 
 ```bash
-node example-usage.js
+# 8 GB RAM
+node --max-old-space-size=8192 example-usage.js
 ```
 
 ## 📤 Output

package/example-usage.js

@@ -101,7 +101,9 @@ async function electronicintifada ()
 			"https://electronicintifada.net/blog",
 			"https://electronicintifada.net/people",
 			"https://electronicintifada.net/location",
-			"https://electronicintifada.net/file"
+			"https://electronicintifada.net/file",
+			"https://electronicintifada.net/bytopic/people",
+			"https://electronicintifada.net/comment/"
 		],
 		exactExcludeList: [
 			"https://electronicintifada.net",
@@ -115,6 +117,7 @@ async function electronicintifada ()
 		textOutputPath: "./dataset/electronicintifada/texts",
 		csvOutputPath: "./dataset/electronicintifada/train.csv",
 		includeMetadata: true,
+		maxArticles: 1000,
 		metadataFields: ["title", "description", "author"]
 	});
 	await scraper.start();

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "clean-web-scraper",
-  "version": "3.4.0",
+  "version": "3.5.0",
   "main": "main.js",
   "scripts": {
     "start": "node main.js",

package/src/WebScraper.js

@@ -13,6 +13,7 @@ class WebScraper
 		baseURL,
 		startURL,
 		maxDepth = Infinity,
+		maxArticles = Infinity, // Add this line
 		excludeList,
 		exactExcludeList,
 		scrapResultPath = "./dataset",
@@ -33,6 +34,7 @@ class WebScraper
 		this.baseURL = baseURL;
 		this.startURL = startURL || baseURL;
 		this.maxDepth = maxDepth;
+		this.maxArticles = maxArticles; // Add this line
 		this.scrapResultPath = scrapResultPath;
 		this.jsonlOutputPath = jsonlOutputPath || path.join( this.scrapResultPath, "train.jsonl" );
 		this.textOutputPath = textOutputPath || path.join( this.scrapResultPath, "texts" );
@@ -113,18 +115,28 @@ class WebScraper
 
 	async fetchPage ( url, depth )
 	{
+		if ( this.allProcessedContent.length >= this.maxArticles )
+		{
+			console.log( `Reached maximum number of articles (${this.maxArticles})` );
+			return;
+		}
 		if ( depth > this.maxDepth )
 		{
 			return;
 		}
 		this.visited.add( url );
+		if ( !this.isValidFileType( url ) )
+		{
+			return;
+		}
 		try
 		{
 			const data = await this.caller( url );
+			if ( !data ) return;
 			const dom = new JSDOM( data, { url });
 			const { document } = dom.window;
 
-			if ( !this.isExcluded( url ) && this.isValidFileType( url ) )
+			if ( !this.isExcluded( url ) )
 			{
 				const reader = new Readability( document, { charThreshold: 500, nbTopCandidates: 20 });
 				const article = reader.parse();