clean-web-scraper 3.5.1 → 3.5.2

package/example-usage.js

@@ -8,8 +8,36 @@ const WebScraper = require( "./src/WebScraper" );
 // 	"Cookie": cookies
 // }
 
+async function palianswers ( enable )
+{
+	const scraper = new WebScraper({
+		baseURL: "https://palianswers.com",
+		excludeList: [
+			"https://palianswers.com/chat/",
+			"https://palianswers.com/become-a-volunteer/",
+			"https://palianswers.com/other-resources/",
+			"https://palianswers.com/request-a-rebuttal/",
+			"https://palianswers.com/submit-a-rebuttal/",
+			"https://palianswers.com/themes/"
+		],
+		exactExcludeList: [
+			"https://palianswers.com/",
+		],
+		scrapResultPath: "./dataset/palianswers/website",
+		jsonlOutputPath: "./dataset/palianswers/train.jsonl",
+		textOutputPath: "./dataset/palianswers/texts",
+		csvOutputPath: "./dataset/palianswers/train.csv",
+		includeMetadata: true,
+		metadataFields: ["title", "description", "author"]
+	});
+	if ( enable )
+	{
+		await scraper.start();
+	}
+	return scraper;
+}
 
-async function khameneiIrFreePalestineTag ()
+async function khameneiIrFreePalestineTag ( enable )
 {
 	// https://english.khamenei.ir/Opinions/FreePalestine
 	// https://english.khamenei.ir/page/search.xhtml?topicid=0&period=0&q=FreePalestine&pageSize=100#
@@ -135,6 +163,7 @@ async function electronicintifada ( enable )
 
 async function palestineremembered ( enable )
 {
+	// https://www.palestineremembered.com
 	const scraper = new WebScraper({
 		baseURL: "https://www.palestineremembered.com",
 		startURL: "https://www.palestineremembered.com/ZionistFAQ.html",
@@ -179,24 +208,21 @@ async function palestineremembered ( enable )
 
 void async function main ()
 {
-	const khameneiIrFreePalestineTagScraper = await khameneiIrFreePalestineTag( false );
-	const decolonizepalestineScraper = await decolonizepalestine( false );
+	const palianswersScraper = await palianswers( true );
+	const decolonizepalestineScraper = await decolonizepalestine( true );
+	const khameneiIrFreePalestineTagScraper = await khameneiIrFreePalestineTag( true );
 	const bdsmovementScraper = await bdsmovement( false );
-	const electronicintifadaScraper = await electronicintifada( false );
-	  const palestinerememberedScraper = await palestineremembered( true );
+	const electronicintifadaScraper = await electronicintifada( true );
+	const palestinerememberedScraper = await palestineremembered( false );
+
 	await WebScraper.combineResults( "./dataset/combined", [
-		khameneiIrFreePalestineTagScraper,
+		palianswersScraper,
 		decolonizepalestineScraper,
-		bdsmovementScraper,
+		khameneiIrFreePalestineTagScraper,
 		electronicintifadaScraper,
-		palestinerememberedScraper
+		// bdsmovementScraper,
+		// palestinerememberedScraper,
 	] );
 
-	// 5
-	// https://www.palestineremembered.com/ZionistFAQ.html
-
-	// 6 https://the-palestinian-side.vercel.app/
-
 	// 7 https://stand-with-palestine.org/blogs
 }()
-

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "clean-web-scraper",
-  "version": "3.5.1",
+  "version": "3.5.2",
   "main": "main.js",
   "scripts": {
     "start": "node main.js",

package/src/WebScraper.js

@@ -12,8 +12,9 @@ class WebScraper
 	constructor ({
 		baseURL,
 		startURL,
+		strictBaseURL = true,
 		maxDepth = Infinity,
-		maxArticles = Infinity, // Add this line
+		maxArticles = Infinity,
 		excludeList,
 		exactExcludeList,
 		scrapResultPath = "./dataset",
@@ -34,8 +35,9 @@ class WebScraper
 	{
 		this.baseURL = baseURL;
 		this.startURL = startURL || baseURL;
+		this.strictBaseURL = strictBaseURL;
 		this.maxDepth = maxDepth;
-		this.maxArticles = maxArticles; // Add this line
+		this.maxArticles = maxArticles;
 		this.scrapResultPath = scrapResultPath;
 		this.jsonlOutputPath = jsonlOutputPath || path.join( this.scrapResultPath, "train.jsonl" );
 		this.textOutputPath = textOutputPath || path.join( this.scrapResultPath, "texts" );
@@ -131,6 +133,10 @@ class WebScraper
 		{
 			return;
 		}
+		if ( !this.isValidDomain( url ) )
+		{
+			return;
+		}
 		try
 		{
 			const data = await this.caller( url );
@@ -608,6 +614,22 @@ class WebScraper
 		return !this.excludedFileTypes.some( ext => { return urlPath.endsWith( ext ) });
 	}
 
+	isValidDomain ( url )
+	{
+		if ( !this.strictBaseURL ) return true;
+		try
+		{
+			const urlObj = new URL( url );
+			const baseURLObj = new URL( this.baseURL );
+			return urlObj.hostname === baseURLObj.hostname;
+		}
+		catch ( e )
+		{
+			console.log( `Invalid URL: ${url}` );
+			return false;
+		}
+	}
+
 	isValidContent ( content )
 	{
 		// Remove whitespace and newlines for checking