clean-web-scraper 3.4.1 → 3.5.1

package/README.md

@@ -89,7 +89,8 @@ await WebScraper.combineResults('./combined', [docsScraper, blogScraper]);
 ```
 
 ```bash
-node example-usage.js
+# 8 GB RAM
+node --max-old-space-size=8192 example-usage.js
 ```
 
 ## 📤 Output

package/example-usage.js

@@ -27,13 +27,15 @@ async function khameneiIrFreePalestineTag ()
 		includeMetadata: true,
 		metadataFields: ["title", "description", "author"]
 	});
-	await scraper.start();
+	if ( enable )
+	{
+		await scraper.start();
+	}
 	return scraper;
 }
 
-async function decolonizepalestine ()
+async function decolonizepalestine ( enable )
 {
-	// https://decolonizepalestine.com
 	const scraper = new WebScraper({
 		baseURL: "https://decolonizepalestine.com",
 		excludeList: [
@@ -54,13 +56,15 @@ async function decolonizepalestine ()
 		includeMetadata: true,
 		metadataFields: ["title", "description", "author"]
 	});
-	await scraper.start();
+	if ( enable )
+	{
+		await scraper.start();
+	}
 	return scraper;
 }
 
-async function bdsmovement ()
+async function bdsmovement ( enable )
 {
-	// https://bdsmovement.net
 	const scraper = new WebScraper({
 		baseURL: "https://bdsmovement.net",
 		excludeList: [
@@ -83,15 +87,16 @@ async function bdsmovement ()
 			host: "socks5://127.0.0.1",
 			port: "2080",
 		},
-		// usePuppeteer: true
 	});
-	await scraper.start();
+	if ( enable )
+	{
+		await scraper.start();
+	}
 	return scraper;
 }
 
-async function electronicintifada ()
+async function electronicintifada ( enable )
 {
-	// https://electronicintifada.net
 	const scraper = new WebScraper({
 		baseURL: "https://electronicintifada.net",
 		excludeList: [
@@ -101,7 +106,10 @@ async function electronicintifada ()
 			"https://electronicintifada.net/blog",
 			"https://electronicintifada.net/people",
 			"https://electronicintifada.net/location",
-			"https://electronicintifada.net/file"
+			"https://electronicintifada.net/file",
+			"https://electronicintifada.net/bytopic/people",
+			"https://electronicintifada.net/comment/",
+			"https://electronicintifada.net/search/site/"
 		],
 		exactExcludeList: [
 			"https://electronicintifada.net",
@@ -115,23 +123,73 @@ async function electronicintifada ()
 		textOutputPath: "./dataset/electronicintifada/texts",
 		csvOutputPath: "./dataset/electronicintifada/train.csv",
 		includeMetadata: true,
+		maxArticles: 2000,
 		metadataFields: ["title", "description", "author"]
 	});
-	await scraper.start();
+	if ( enable )
+	{
+		await scraper.start();
+	}
 	return scraper;
 }
 
+async function palestineremembered ( enable )
+{
+	const scraper = new WebScraper({
+		baseURL: "https://www.palestineremembered.com",
+		startURL: "https://www.palestineremembered.com/ZionistFAQ.html",
+		excludeList: [
+			"https://www.palestineremembered.com/GeoPoints",
+			"https://www.palestineremembered.com/Donate",
+			"https://www.palestineremembered.com/ContactUs.html",
+			"https://www.palestineremembered.com/tags/Looting-Palestinian-properties.html",
+			"https://www.palestineremembered.com/ar/",
+			"https://www.palestineremembered.com/OldNewPictures.html",
+			"https://www.palestineremembered.com/Maps/index.html",
+			"https://www.palestineremembered.com/OralHistory/Interviews-Listing/",
+			"https://www.palestineremembered.com/Acre/Famous-Zionist-Quotes/Story637.html",
+			"https://www.palestineremembered.com/Articles/General/Story2045.html",
+			"https://www.palestineremembered.com/AllTownsListing.html",
+			"https://www.palestineremembered.com/Articles/General/ar/",
+			"https://www.palestineremembered.com/SiteVideos.html"
+		],
+		exactExcludeList: [
+			"https://www.palestineremembered.com/index.html",
+			"https://www.palestineremembered.com/ZionistFAQ.html"
+		],
+		scrapResultPath: "./dataset/palestineremembered/website",
+		jsonlOutputPath: "./dataset/palestineremembered/train.jsonl",
+		textOutputPath: "./dataset/palestineremembered/texts",
+		csvOutputPath: "./dataset/palestineremembered/train.csv",
+		includeMetadata: true,
+		metadataFields: ["title", "description", "author"],
+		axiosProxy: {
+			host: "localhost",
+			port: 2080,
+			protocol: "http"
+		}
+	});
+	if ( enable )
+	{
+		await scraper.start();
+	}
+	return scraper;
+}
+
+
 void async function main ()
 {
-	const khameneiIrFreePalestineTagScraper = await khameneiIrFreePalestineTag();
-	const decolonizepalestineScraper = await decolonizepalestine();
-	const bdsmovementScraper = await bdsmovement();
-	const electronicintifadaScraper = await electronicintifada();
+	const khameneiIrFreePalestineTagScraper = await khameneiIrFreePalestineTag( false );
+	const decolonizepalestineScraper = await decolonizepalestine( false );
+	const bdsmovementScraper = await bdsmovement( false );
+	const electronicintifadaScraper = await electronicintifada( false );
+	  const palestinerememberedScraper = await palestineremembered( true );
 	await WebScraper.combineResults( "./dataset/combined", [
 		khameneiIrFreePalestineTagScraper,
 		decolonizepalestineScraper,
 		bdsmovementScraper,
-		electronicintifadaScraper
+		electronicintifadaScraper,
+		palestinerememberedScraper
 	] );
 
 	// 5

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "clean-web-scraper",
-  "version": "3.4.1",
+  "version": "3.5.1",
   "main": "main.js",
   "scripts": {
     "start": "node main.js",

package/src/WebScraper.js

@@ -13,6 +13,7 @@ class WebScraper
 		baseURL,
 		startURL,
 		maxDepth = Infinity,
+		maxArticles = Infinity, // Add this line
 		excludeList,
 		exactExcludeList,
 		scrapResultPath = "./dataset",
@@ -21,7 +22,8 @@ class WebScraper
 		csvOutputPath,
 		includeMetadata = false,
 		metadataFields = [], // ['title', 'description', 'author', etc.]
-		headers,
+		axiosHeaders,
+		axiosProxy,
 		usePuppeteer,
 		puppeteerProxy, // e.g. http://127.0.0.1:2080
 		puppeteerExecutablePath,
@@ -33,13 +35,14 @@ class WebScraper
 		this.baseURL = baseURL;
 		this.startURL = startURL || baseURL;
 		this.maxDepth = maxDepth;
+		this.maxArticles = maxArticles; // Add this line
 		this.scrapResultPath = scrapResultPath;
 		this.jsonlOutputPath = jsonlOutputPath || path.join( this.scrapResultPath, "train.jsonl" );
 		this.textOutputPath = textOutputPath || path.join( this.scrapResultPath, "texts" );
 		this.csvOutputPath = csvOutputPath || path.join( this.scrapResultPath, "train.csv" );
 		this.jsonlOutputPathWithMeta = jsonlOutputPath.replace( ".jsonl", "_with_metadata.jsonl" );
 		this.csvOutputPathWithMeta = csvOutputPath.replace( ".csv", "_with_metadata.csv" );
-		this.headers = headers;
+		this.axiosHeaders = axiosHeaders;
 		this.includeMetadata = includeMetadata;
 	   this.metadataFields = new Set( metadataFields );
 		this.visited = new Set();
@@ -48,6 +51,7 @@ class WebScraper
 		this.allProcessedContent = [];
 		this.filterFileTypes = filterFileTypes;
 		this.excludedFileTypes = excludedFileTypes;
+		this.axiosProxy = axiosProxy;
 		this.usePuppeteer = usePuppeteer || false;
 		this.puppeteerOptions = {
 			headless: false,
@@ -113,6 +117,11 @@ class WebScraper
 
 	async fetchPage ( url, depth )
 	{
+		if ( this.allProcessedContent.length >= this.maxArticles )
+		{
+			console.log( `Reached maximum number of articles (${this.maxArticles})` );
+			return;
+		}
 		if ( depth > this.maxDepth )
 		{
 			return;
@@ -125,6 +134,7 @@ class WebScraper
 		try
 		{
 			const data = await this.caller( url );
+			if ( !data ) return;
 			const dom = new JSDOM( data, { url });
 			const { document } = dom.window;
 
@@ -172,9 +182,13 @@ class WebScraper
 		try
 		{
 			let axiosOptions = {};
-			if ( this.headers )
+			if ( this.axiosHeaders )
+			{
+				axiosOptions.headers = this.axiosHeaders;
+			}
+			if ( this.axiosProxy )
 			{
-				axiosOptions.headers = this.headers;
+				axiosOptions.proxy = this.axiosProxy;
 			}
 
 			// Step 1: Make a GET request with a small timeout and limited data download