clean-web-scraper 3.5.0 → 3.5.2

package/example-usage.js

@@ -8,8 +8,36 @@ const WebScraper = require( "./src/WebScraper" );
 // 	"Cookie": cookies
 // }
 
+async function palianswers ( enable )
+{
+	const scraper = new WebScraper({
+		baseURL: "https://palianswers.com",
+		excludeList: [
+			"https://palianswers.com/chat/",
+			"https://palianswers.com/become-a-volunteer/",
+			"https://palianswers.com/other-resources/",
+			"https://palianswers.com/request-a-rebuttal/",
+			"https://palianswers.com/submit-a-rebuttal/",
+			"https://palianswers.com/themes/"
+		],
+		exactExcludeList: [
+			"https://palianswers.com/",
+		],
+		scrapResultPath: "./dataset/palianswers/website",
+		jsonlOutputPath: "./dataset/palianswers/train.jsonl",
+		textOutputPath: "./dataset/palianswers/texts",
+		csvOutputPath: "./dataset/palianswers/train.csv",
+		includeMetadata: true,
+		metadataFields: ["title", "description", "author"]
+	});
+	if ( enable )
+	{
+		await scraper.start();
+	}
+	return scraper;
+}
 
-async function khameneiIrFreePalestineTag ()
+async function khameneiIrFreePalestineTag ( enable )
 {
 	// https://english.khamenei.ir/Opinions/FreePalestine
 	// https://english.khamenei.ir/page/search.xhtml?topicid=0&period=0&q=FreePalestine&pageSize=100#
@@ -27,13 +55,15 @@ async function khameneiIrFreePalestineTag ()
 		includeMetadata: true,
 		metadataFields: ["title", "description", "author"]
 	});
-	await scraper.start();
+	if ( enable )
+	{
+		await scraper.start();
+	}
 	return scraper;
 }
 
-async function decolonizepalestine ()
+async function decolonizepalestine ( enable )
 {
-	// https://decolonizepalestine.com
 	const scraper = new WebScraper({
 		baseURL: "https://decolonizepalestine.com",
 		excludeList: [
@@ -54,13 +84,15 @@ async function decolonizepalestine ()
 		includeMetadata: true,
 		metadataFields: ["title", "description", "author"]
 	});
-	await scraper.start();
+	if ( enable )
+	{
+		await scraper.start();
+	}
 	return scraper;
 }
 
-async function bdsmovement ()
+async function bdsmovement ( enable )
 {
-	// https://bdsmovement.net
 	const scraper = new WebScraper({
 		baseURL: "https://bdsmovement.net",
 		excludeList: [
@@ -83,15 +115,16 @@ async function bdsmovement ()
 			host: "socks5://127.0.0.1",
 			port: "2080",
 		},
-		// usePuppeteer: true
 	});
-	await scraper.start();
+	if ( enable )
+	{
+		await scraper.start();
+	}
 	return scraper;
 }
 
-async function electronicintifada ()
+async function electronicintifada ( enable )
 {
-	// https://electronicintifada.net
 	const scraper = new WebScraper({
 		baseURL: "https://electronicintifada.net",
 		excludeList: [
@@ -103,7 +136,8 @@ async function electronicintifada ()
 			"https://electronicintifada.net/location",
 			"https://electronicintifada.net/file",
 			"https://electronicintifada.net/bytopic/people",
-			"https://electronicintifada.net/comment/"
+			"https://electronicintifada.net/comment/",
+			"https://electronicintifada.net/search/site/"
 		],
 		exactExcludeList: [
 			"https://electronicintifada.net",
@@ -117,31 +151,78 @@ async function electronicintifada ()
 		textOutputPath: "./dataset/electronicintifada/texts",
 		csvOutputPath: "./dataset/electronicintifada/train.csv",
 		includeMetadata: true,
-		maxArticles: 1000,
+		maxArticles: 2000,
 		metadataFields: ["title", "description", "author"]
 	});
-	await scraper.start();
+	if ( enable )
+	{
+		await scraper.start();
+	}
+	return scraper;
+}
+
+async function palestineremembered ( enable )
+{
+	// https://www.palestineremembered.com
+	const scraper = new WebScraper({
+		baseURL: "https://www.palestineremembered.com",
+		startURL: "https://www.palestineremembered.com/ZionistFAQ.html",
+		excludeList: [
+			"https://www.palestineremembered.com/GeoPoints",
+			"https://www.palestineremembered.com/Donate",
+			"https://www.palestineremembered.com/ContactUs.html",
+			"https://www.palestineremembered.com/tags/Looting-Palestinian-properties.html",
+			"https://www.palestineremembered.com/ar/",
+			"https://www.palestineremembered.com/OldNewPictures.html",
+			"https://www.palestineremembered.com/Maps/index.html",
+			"https://www.palestineremembered.com/OralHistory/Interviews-Listing/",
+			"https://www.palestineremembered.com/Acre/Famous-Zionist-Quotes/Story637.html",
+			"https://www.palestineremembered.com/Articles/General/Story2045.html",
+			"https://www.palestineremembered.com/AllTownsListing.html",
+			"https://www.palestineremembered.com/Articles/General/ar/",
+			"https://www.palestineremembered.com/SiteVideos.html"
+		],
+		exactExcludeList: [
+			"https://www.palestineremembered.com/index.html",
+			"https://www.palestineremembered.com/ZionistFAQ.html"
+		],
+		scrapResultPath: "./dataset/palestineremembered/website",
+		jsonlOutputPath: "./dataset/palestineremembered/train.jsonl",
+		textOutputPath: "./dataset/palestineremembered/texts",
+		csvOutputPath: "./dataset/palestineremembered/train.csv",
+		includeMetadata: true,
+		metadataFields: ["title", "description", "author"],
+		axiosProxy: {
+			host: "localhost",
+			port: 2080,
+			protocol: "http"
+		}
+	});
+	if ( enable )
+	{
+		await scraper.start();
+	}
 	return scraper;
 }
 
+
 void async function main ()
 {
-	const khameneiIrFreePalestineTagScraper = await khameneiIrFreePalestineTag();
-	const decolonizepalestineScraper = await decolonizepalestine();
-	const bdsmovementScraper = await bdsmovement();
-	const electronicintifadaScraper = await electronicintifada();
+	const palianswersScraper = await palianswers( true );
+	const decolonizepalestineScraper = await decolonizepalestine( true );
+	const khameneiIrFreePalestineTagScraper = await khameneiIrFreePalestineTag( true );
+	const bdsmovementScraper = await bdsmovement( false );
+	const electronicintifadaScraper = await electronicintifada( true );
+	const palestinerememberedScraper = await palestineremembered( false );
+
 	await WebScraper.combineResults( "./dataset/combined", [
-		khameneiIrFreePalestineTagScraper,
+		palianswersScraper,
 		decolonizepalestineScraper,
-		bdsmovementScraper,
-		electronicintifadaScraper
+		khameneiIrFreePalestineTagScraper,
+		electronicintifadaScraper,
+		// bdsmovementScraper,
+		// palestinerememberedScraper,
 	] );
 
-	// 5
-	// https://www.palestineremembered.com/ZionistFAQ.html
-
-	// 6 https://the-palestinian-side.vercel.app/
-
 	// 7 https://stand-with-palestine.org/blogs
 }()
-

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "clean-web-scraper",
-  "version": "3.5.0",
+  "version": "3.5.2",
   "main": "main.js",
   "scripts": {
     "start": "node main.js",

package/src/WebScraper.js

@@ -12,8 +12,9 @@ class WebScraper
 	constructor ({
 		baseURL,
 		startURL,
+		strictBaseURL = true,
 		maxDepth = Infinity,
-		maxArticles = Infinity, // Add this line
+		maxArticles = Infinity,
 		excludeList,
 		exactExcludeList,
 		scrapResultPath = "./dataset",
@@ -22,7 +23,8 @@ class WebScraper
 		csvOutputPath,
 		includeMetadata = false,
 		metadataFields = [], // ['title', 'description', 'author', etc.]
-		headers,
+		axiosHeaders,
+		axiosProxy,
 		usePuppeteer,
 		puppeteerProxy, // e.g. http://127.0.0.1:2080
 		puppeteerExecutablePath,
@@ -33,15 +35,16 @@ class WebScraper
 	{
 		this.baseURL = baseURL;
 		this.startURL = startURL || baseURL;
+		this.strictBaseURL = strictBaseURL;
 		this.maxDepth = maxDepth;
-		this.maxArticles = maxArticles; // Add this line
+		this.maxArticles = maxArticles;
 		this.scrapResultPath = scrapResultPath;
 		this.jsonlOutputPath = jsonlOutputPath || path.join( this.scrapResultPath, "train.jsonl" );
 		this.textOutputPath = textOutputPath || path.join( this.scrapResultPath, "texts" );
 		this.csvOutputPath = csvOutputPath || path.join( this.scrapResultPath, "train.csv" );
 		this.jsonlOutputPathWithMeta = jsonlOutputPath.replace( ".jsonl", "_with_metadata.jsonl" );
 		this.csvOutputPathWithMeta = csvOutputPath.replace( ".csv", "_with_metadata.csv" );
-		this.headers = headers;
+		this.axiosHeaders = axiosHeaders;
 		this.includeMetadata = includeMetadata;
 	   this.metadataFields = new Set( metadataFields );
 		this.visited = new Set();
@@ -50,6 +53,7 @@ class WebScraper
 		this.allProcessedContent = [];
 		this.filterFileTypes = filterFileTypes;
 		this.excludedFileTypes = excludedFileTypes;
+		this.axiosProxy = axiosProxy;
 		this.usePuppeteer = usePuppeteer || false;
 		this.puppeteerOptions = {
 			headless: false,
@@ -129,6 +133,10 @@ class WebScraper
 		{
 			return;
 		}
+		if ( !this.isValidDomain( url ) )
+		{
+			return;
+		}
 		try
 		{
 			const data = await this.caller( url );
@@ -180,9 +188,13 @@ class WebScraper
 		try
 		{
 			let axiosOptions = {};
-			if ( this.headers )
+			if ( this.axiosHeaders )
 			{
-				axiosOptions.headers = this.headers;
+				axiosOptions.headers = this.axiosHeaders;
+			}
+			if ( this.axiosProxy )
+			{
+				axiosOptions.proxy = this.axiosProxy;
 			}
 
 			// Step 1: Make a GET request with a small timeout and limited data download
@@ -602,6 +614,22 @@ class WebScraper
 		return !this.excludedFileTypes.some( ext => { return urlPath.endsWith( ext ) });
 	}
 
+	isValidDomain ( url )
+	{
+		if ( !this.strictBaseURL ) return true;
+		try
+		{
+			const urlObj = new URL( url );
+			const baseURLObj = new URL( this.baseURL );
+			return urlObj.hostname === baseURLObj.hostname;
+		}
+		catch ( e )
+		{
+			console.log( `Invalid URL: ${url}` );
+			return false;
+		}
+	}
+
 	isValidContent ( content )
 	{
 		// Remove whitespace and newlines for checking