clean-web-scraper 4.2.2 → 4.3.0

package/README.md

@@ -91,7 +91,7 @@ const docsScraper = new WebScraper({
   scrapResultPath: './datasets/docs',
   maxDepth: 3,                               // Optional: Maximum depth for recursive crawling
   includeMetadata: true,                     // Optional: Include metadata in output files
-  metadataFields: ["author", "articleTitle", "pageTitle", "description", "dateScrapedDate"],
+  metadataFields: ["author", "articleTitle", "pageTitle", "description", "dataScrapedDate"],
    // Optional: Specify metadata fields to include
 });
 
@@ -101,7 +101,7 @@ const blogScraper = new WebScraper({
   scrapResultPath: './datasets/blog',
   maxDepth: 3,                               // Optional: Maximum depth for recursive crawling
   includeMetadata: true,                     // Optional: Include metadata in output files
-  metadataFields: ["author", "articleTitle", "pageTitle", "description", "dateScrapedDate"],
+  metadataFields: ["author", "articleTitle", "pageTitle", "description", "dataScrapedDate"],
    // Optional: Specify metadata fields to include
 });
 

package/example-usage.js

@@ -37,9 +37,8 @@ async function palianswers ( enable )
 		textOutputPath: "./dataset/palianswers/texts",
 		csvOutputPath: "./dataset/palianswers/train.csv",
 		includeMetadata: true,
-		metadataFields: ["author", "articleTitle", "pageTitle", "description", "dateScrapedDate"],
+		metadataFields: ["author", "articleTitle", "pageTitle", "description", "dataScrapedDate", "url"],
 		axiosRetryDelay: 10000,
-		crawlingDelay: 0
 	};
 	return await runScraper( config, enable );
 }
@@ -50,7 +49,7 @@ async function khameneiIrFreePalestineTag ( enable )
 		baseURL: "https://english.khamenei.ir/news",
 		startURL: "https://english.khamenei.ir/page/search.xhtml?topicid=0&period=0&q=FreePalestine&pageSize=100#",
 		maxDepth: 1,
-		maxArticles: 2,
+		maxArticles: 200,
 		exactExcludeList: [
 			"https://english.khamenei.ir/page/search.xhtml?topicid=0&period=0&q=FreePalestine&pageSize=100#",
 			"https://english.khamenei.ir/page/search.xhtml?topicid=0&period=0&q=FreePalestine&pageSize=100"
@@ -60,7 +59,7 @@ async function khameneiIrFreePalestineTag ( enable )
 		textOutputPath: "./dataset/khamenei-ir-free-palestine-tag/texts",
 		csvOutputPath: "./dataset/khamenei-ir-free-palestine-tag/train.csv",
 		includeMetadata: true,
-		metadataFields: ["author", "articleTitle", "pageTitle", "description", "dateScrapedDate"],
+		metadataFields: ["author", "articleTitle", "pageTitle", "description", "dataScrapedDate", "url"],
 		axiosRetryDelay: 10000,
 	};
 	return await runScraper( config, enable );
@@ -72,8 +71,8 @@ async function khameneiIrPalestineSpecialPage ( enable )
 	const config = {
 		baseURL: "https://english.khamenei.ir/news",
 		startURL: "https://english.khamenei.ir/palestine-special-page",
-		maxDepth: 2,
-		maxArticles: 2,
+		maxDepth: 1,
+		maxArticles: 200,
 		exactExcludeList: [
 			"https://english.khamenei.ir/palestine-special-page/"
 		],
@@ -82,7 +81,7 @@ async function khameneiIrPalestineSpecialPage ( enable )
 		textOutputPath: "./dataset/khamenei-ir-palestine-special-page/texts",
 		csvOutputPath: "./dataset/khamenei-ir-palestine-special-page/train.csv",
 		includeMetadata: true,
-		metadataFields: ["author", "articleTitle", "pageTitle", "description", "dateScrapedDate"],
+		metadataFields: ["author", "articleTitle", "pageTitle", "description", "dataScrapedDate", "url"],
 		axiosRetryDelay: 10000
 	};
 	return await runScraper( config, enable );
@@ -103,13 +102,13 @@ async function decolonizepalestine ( enable )
 			"https://decolonizepalestine.com/rainbow-washing",
 			"https://decolonizepalestine.com/"
 		],
-		maxArticles: 2,
+		maxArticles: 400,
 		scrapResultPath: "./dataset/decolonizepalestine/website",
 		jsonlOutputPath: "./dataset/decolonizepalestine/train.jsonl",
 		textOutputPath: "./dataset/decolonizepalestine/texts",
 		csvOutputPath: "./dataset/decolonizepalestine/train.csv",
 		includeMetadata: true,
-		metadataFields: ["author", "articleTitle", "pageTitle", "description", "dateScrapedDate"],
+		metadataFields: ["author", "articleTitle", "pageTitle", "description", "dataScrapedDate", "url"],
 		axiosRetryDelay: 10000,
 	};
 	return await runScraper( config, enable );
@@ -123,7 +122,6 @@ async function electronicintifada ( enable )
 			"https://electronicintifada.net/updates",
 			"https://electronicintifada.net/taxonomy/term/",
 			"https://electronicintifada.net/tags/",
-			"https://electronicintifada.net/blog",
 			"https://electronicintifada.net/people",
 			"https://electronicintifada.net/location",
 			"https://electronicintifada.net/file",
@@ -134,11 +132,13 @@ async function electronicintifada ( enable )
 			"https://electronicintifada.net/opinion",
 			"https://electronicintifada.net/about-ei",
 			"https://electronicintifada.net/review",
-			"https://electronicintifada.net/artmusicculture"
+			"https://electronicintifada.net/artmusicculture",
+			"https://electronicintifada.net/blog/editors",
+			"https://electronicintifada.net/blog"
 		],
 		exactExcludeList: [
 			"https://electronicintifada.net",
-			"https://electronicintifada.net/blog",
+			"https://electronicintifada.net/blogs",
 			"https://electronicintifada.net/review",
 		],
 		scrapResultPath: "./dataset/electronicintifada/website",
@@ -146,11 +146,12 @@ async function electronicintifada ( enable )
 		textOutputPath: "./dataset/electronicintifada/texts",
 		csvOutputPath: "./dataset/electronicintifada/train.csv",
 		includeMetadata: true,
-		metadataFields: ["author", "articleTitle", "pageTitle", "description", "dateScrapedDate"],
-		maxDepth: 16,
+		metadataFields: ["author", "articleTitle", "pageTitle", "description", "dataScrapedDate", "url"],
 		maxArticles: 2000,
+		maxDepth: 16,
+		batchSize: 30,
 		axiosHeaders: headers,
-		axiosMaxRetries: 2,
+		axiosMaxRetries: 3,
 		axiosRetryDelay: 10000,
 		axiosProxy: {
 			host: "localhost",
@@ -158,7 +159,6 @@ async function electronicintifada ( enable )
 			protocol: "http"
 		},
 		useProxyAsFallback: true,
-		crawlingDelay: 1
 	};
 	return await runScraper( config, enable );
 }
@@ -175,7 +175,7 @@ async function standWithPalestine ( enable )
 		csvOutputPath: "./dataset/stand-with-palestine/train.csv",
 		axiosHeaders: headers,
 		includeMetadata: true,
-		metadataFields: ["author", "articleTitle", "pageTitle", "description", "dateScrapedDate"]
+		metadataFields: ["author", "articleTitle", "pageTitle", "description", "dataScrapedDate", "url"]
 	};
 	return await runScraper( config, enable );
 }
@@ -211,20 +211,24 @@ async function mondoweiss ( enable )
 			"https://mondoweiss.net/activism/",
 			"https://mondoweiss.net/news-letters/",
 			"https://mondoweiss.net/newsletters",
-			/^https:\/\/mondoweiss\.net\/\d{4}\/\d{2}$/,
-			/^https:\/\/mondoweiss\.net\/\d{4}\/$/,
 			"https://mondoweiss.net/daily-headlines",
 			"https://mondoweiss.net/palestineletter",
 			"https://mondoweiss.net/podcasts/",
 			"https://mondoweiss.net/the-shift",
-			"https://mondoweiss.net/weekly-briefing"
+			"https://mondoweiss.net/weekly-briefing",
+			"https://mondoweiss.net/contact/",
+			/^https:\/\/mondoweiss\.net\/\d{4}\/\d{2}\/?$/,
+			/^https:\/\/mondoweiss\.net\/\d{4}\/?$/
 		],
 		scrapResultPath: "./dataset/mondoweiss/website",
 		jsonlOutputPath: "./dataset/mondoweiss/train.jsonl",
 		textOutputPath: "./dataset/mondoweiss/texts",
 		csvOutputPath: "./dataset/mondoweiss/train.csv",
+		includeMetadata: true,
+		metadataFields: ["author", "articleTitle", "pageTitle", "description", "dataScrapedDate", "url"],
 		maxArticles: 2500,
 		maxDepth: 15,
+		batchSize: 20,
 		axiosHeaders: headers,
 		axiosMaxRetries: 3,
 		axiosRetryDelay: 10000,
@@ -234,10 +238,6 @@ async function mondoweiss ( enable )
 			protocol: "http"
 		},
 		useProxyAsFallback: true,
-		includeMetadata: true,
-		metadataFields: ["author", "articleTitle", "pageTitle", "description", "dateScrapedDate"],
-		crawlingDelay: 0,
-		batchSize: 20
 	};
 	return await runScraper( config, enable );
 }
@@ -251,19 +251,36 @@ async function bdsmovement ( enable )
 			"https://bdsmovement.net/privacy-policy",
 			"https://bdsmovement.net/get-involved/join-a-bds-campaign",
 			"https://bdsmovement.net/donate_",
+			"https://bdsmovement.net/donate",
 			"https://bdsmovement.net/user",
-			"https://bdsmovement.net/admin"
+			"https://bdsmovement.net/admin",
+			"https://bdsmovement.net/stay-updated",
+			"https://bdsmovement.net/join-a-bds-campaign",
+			"https://bdsmovement.net/contact-us",
+			"https://bdsmovement.net/taxonomy",
+			"https://bdsmovement.net/news-type",
+			"https://bdsmovement.net/cdn-cgi"
+		],
+		exactExcludeList: [
+			"https://bdsmovement.net/",
+			"https://bdsmovement.net/shutdownnation",
+			"https://bdsmovement.net/campaigns",
+			"https://bdsmovement.net/resources",
+			/^https:\/\/bdsmovement\.net\/resources\?page=\d+$/,
+			/^https:\/\/bdsmovement\.net\/resources\?campaign=\d+$/,
+			/^https:\/\/bdsmovement\.net\/resources\?type=\d+$/,
+			/^https:\/\/bdsmovement\.net\/news\?type=\d+$/,
+			/^https:\/\/bdsmovement\.net\/news\?campaign=\d+$/,
 		],
 		scrapResultPath: "./dataset/bdsmovement/website",
 		jsonlOutputPath: "./dataset/bdsmovement/train.jsonl",
 		textOutputPath: "./dataset/bdsmovement/texts",
 		csvOutputPath: "./dataset/bdsmovement/train.csv",
 		includeMetadata: true,
-		metadataFields: ["author", "articleTitle", "pageTitle", "description", "dateScrapedDate"],
-		puppeteerRealProxy: {
-			host: "socks5://127.0.0.1",
-			port: "2080",
-		},
+		metadataFields: ["author", "articleTitle", "pageTitle", "description", "dataScrapedDate", "url"],
+		maxArticles: 2000,
+		maxDepth: 16,
+		batchSize: 20
 	};
 	return await runScraper( config, enable );
 }
@@ -297,7 +314,8 @@ async function palestineremembered ( enable )
 		textOutputPath: "./dataset/palestineremembered/texts",
 		csvOutputPath: "./dataset/palestineremembered/train.csv",
 		includeMetadata: true,
-		metadataFields: ["author", "articleTitle", "pageTitle", "description", "dateScrapedDate"],
+		metadataFields: ["author", "articleTitle", "pageTitle", "description", "dataScrapedDate", "url"],
+		batchSize: 10,
 		axiosProxy: {
 			host: "localhost",
 			port: 2080,
@@ -309,23 +327,24 @@ async function palestineremembered ( enable )
 
 void async function main ()
 {
-	// const palianswersScraper = await palianswers( true );
+	const palianswersScraper = await palianswers( true );
 	const decolonizepalestineScraper = await decolonizepalestine( true );
 	const khameneiIrFreePalestineTagScraper = await khameneiIrFreePalestineTag( true );
-	// const khameneiIrPalestineSpecialPageScraper = await khameneiIrPalestineSpecialPage( true );
-	// const electronicintifadaScraper = await electronicintifada( true );
-	// const standWithPalestineScraper = await standWithPalestine( true );
-	// const mondoweisScraper = await mondoweiss( true );
-	// const bdsmovementScraper = await bdsmovement( false );
+	const khameneiIrPalestineSpecialPageScraper = await khameneiIrPalestineSpecialPage( true );
+	const electronicintifadaScraper = await electronicintifada( true );
+	const standWithPalestineScraper = await standWithPalestine( true );
+	const mondoweisScraper = await mondoweiss( true );
+	const bdsmovementScraper = await bdsmovement( true );
 	// const palestinerememberedScraper = await palestineremembered( false );
 
 	await WebScraper.combineResults( "./dataset/combined", [
-		// palianswersScraper,
+		palianswersScraper,
 		decolonizepalestineScraper,
 		khameneiIrFreePalestineTagScraper,
-		// khameneiIrPalestineSpecialPageScraper,
-		// electronicintifadaScraper,
-		// standWithPalestineScraper,
-		// mondoweisScraper
+		khameneiIrPalestineSpecialPageScraper,
+		electronicintifadaScraper,
+		standWithPalestineScraper,
+		mondoweisScraper,
+		bdsmovementScraper,
 	] );
 }();

package/main.js

@@ -18,6 +18,7 @@ class WebScraper
 		this.maxArticles = config.maxArticles || Infinity;
 		this.crawlingDelay = config.crawlingDelay ?? 1000;
 		this.batchSize = config.batchSize || 5;
+		this.minContentLength = config.minContentLength || 400;
 
 		// Output paths setup
 		this.scrapResultPath = config.scrapResultPath || "./dataset";
@@ -469,7 +470,7 @@ class WebScraper
 		let processed = content;
 		// Remove unwanted fixed text
 		processed = processed.replace( /\[You can read more about this here\]/g, "" ).trim();
-		// Trim each line and remove extra newlines
+		processed = processed.replace( /Click on the image to view the large size/g, "" );
 		processed = processed
 		.split( "\n" )
 		.map( line => { return line.trim() })
@@ -525,7 +526,7 @@ class WebScraper
 			ogDescription: document.querySelector( "meta[property=\"og:description\"]" )?.content,
 			ogImage: document.querySelector( "meta[property=\"og:image\"]" )?.content,
 			ogType: document.querySelector( "meta[property=\"og:type\"]" )?.content,
-			dateScrapedDate: new Date().toISOString(),
+			dataScrapedDate: new Date().toISOString(),
 			originalHtml: html,
 		};
 	}
@@ -663,7 +664,7 @@ class WebScraper
 
 		const hasInvalidPhrases = invalidPhrases.some( phrase => { return cleanContent.includes( phrase ) });
 		// Check content length
-		if ( cleanContent.length < 100 || hasInvalidPhrases )
+		if ( cleanContent.length < this.minContentLength || hasInvalidPhrases )
 		{
 			return false;
 		}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "clean-web-scraper",
-  "version": "4.2.2",
+  "version": "4.3.0",
   "main": "main.js",
   "scripts": {
     "start": "node main.js",
@@ -24,10 +24,10 @@
   "description": "",
   "dependencies": {
     "@mozilla/readability": "^0.6.0",
-    "axios": "^1.7.9",
-    "eslint": "^9.17.0",
+    "axios": "^1.8.4",
+    "eslint": "^9.23.0",
     "jsdom": "^26.0.0",
     "puppeteer": "^24.1.1",
     "puppeteer-real-browser": "^1.3.22"
   }
-}
+}