clean-web-scraper 3.2.5 → 3.3.1

package/README.md

@@ -98,7 +98,7 @@ Your AI-ready content is saved in a clean, structured format:
 - 📁 Base folder: `./folderPath/example.com/`
 - 📑 Files preserve original URL paths
 - 🤖 No HTML, no noise - just clean, structured text (`.txt` files)
-- 📊 `JSONL` and `CSV` outputs, ready for AI consumption and model training
+- 📊 `JSONL` and `CSV` outputs, ready for AI consumption, model training and fine-tuning
 
 ```bash
 example.com/
@@ -197,15 +197,6 @@ text,title,author,description
 "Another article","Second Page","Jane Smith","Another description"
 ```
 
-## 🤖 AI/LLM Training Ready
-
-The output is specifically formatted for AI training and fine-tuning purposes:
-
-- Clean, processed text without HTML markup
-- Multiple formats (JSONL, CSV, text files)
-- Structured content perfect for fine-tuning LLMs
-- Ready to use in your ML pipelines
-
 ## Standing with Palestine 🇵🇸
 
 This project supports Palestinian rights and stands in solidarity with Palestine. We believe in the importance of documenting and preserving Palestinian narratives, history, and struggles for justice and liberation.

package/example-usage.js

@@ -3,15 +3,12 @@ const WebScraper = require( "./src/WebScraper" );
 
 async function khameneiIrFreePalestineTag ()
 {
-	// 1
 	// https://english.khamenei.ir/Opinions/FreePalestine
 	// https://english.khamenei.ir/page/search.xhtml?topicid=0&period=0&q=FreePalestine&pageSize=100#
 	const scraper = new WebScraper({
 		baseURL: "https://english.khamenei.ir/news",
 		startURL: "https://english.khamenei.ir/page/search.xhtml?topicid=0&period=0&q=FreePalestine&pageSize=100#",
 		maxDepth: 1,
-		excludeList: [
-		],
 		exactExcludeList: [
 			"https://english.khamenei.ir/page/search.xhtml?topicid=0&period=0&q=FreePalestine&pageSize=100#"
 		],
@@ -20,7 +17,7 @@ async function khameneiIrFreePalestineTag ()
 		textOutputPath: "./dataset/khamenei-ir-free-palestine-tag/texts",
 		csvOutputPath: "./dataset/khamenei-ir-free-palestine-tag/train.csv",
 		includeMetadata: true,
-		metadataFields: ["title", "description", "author", "lastModified", "language"]
+		metadataFields: ["title", "description", "author"]
 	});
 	await scraper.start();
 	return scraper;
@@ -28,7 +25,6 @@ async function khameneiIrFreePalestineTag ()
 
 async function decolonizepalestine ()
 {
-	// 2
 	// https://decolonizepalestine.com
 	const scraper = new WebScraper({
 		baseURL: "https://decolonizepalestine.com",
@@ -48,23 +44,46 @@ async function decolonizepalestine ()
 		textOutputPath: "./dataset/decolonizepalestine/texts",
 		csvOutputPath: "./dataset/decolonizepalestine/train.csv",
 		includeMetadata: true,
-		metadataFields: ["title", "description", "author", "lastModified", "language"]
+		metadataFields: ["title", "description", "author"]
 	});
 	await scraper.start();
 	return scraper;
 }
 
-void async function main ()
+async function bdsmovement ()
 {
-	const khameneiIrFreePalestineTagScraper = await khameneiIrFreePalestineTag();
-	const decolonizepalestineScraper = await decolonizepalestine();
-	await WebScraper.combineResults( "./dataset/combined", [
-		khameneiIrFreePalestineTagScraper,
-		decolonizepalestineScraper
-	] );
+	// https://bdsmovement.org
+	const scraper = new WebScraper({
+		baseURL: "https://bdsmovement.org",
+		excludeList: [
+			"https://bdsmovement.net/press-area",
+			"https://bdsmovement.net/privacy-policy",
+			"https://bdsmovement.net/get-involved/join-a-bds-campaign",
+			"https://bdsmovement.net/donate_",
+			"https://bdsmovement.net/user",
+			"https://bdsmovement.net/admin"
+		],
+		scrapResultPath: "./dataset/bdsmovement/website",
+		jsonlOutputPath: "./dataset/bdsmovement/train.jsonl",
+		textOutputPath: "./dataset/bdsmovement/texts",
+		csvOutputPath: "./dataset/bdsmovement/train.csv",
+		includeMetadata: true,
+		metadataFields: ["title", "description", "author"]
+	});
+	await scraper.start();
+	return scraper;
+}
 
-	// 3
-	// https://bdsmovement.net
+void async function main ()
+{
+	// const khameneiIrFreePalestineTagScraper = await khameneiIrFreePalestineTag();
+	// const decolonizepalestineScraper = await decolonizepalestine();
+	const bdsmovementScraper = await bdsmovement();
+	// await WebScraper.combineResults( "./dataset/combined", [
+	// 	khameneiIrFreePalestineTagScraper,
+	// 	decolonizepalestineScraper,
+	// 	bdsmovementScraper
+	// ] );
 
 	// 4
 	// https://electronicintifada.net/

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "clean-web-scraper",
-  "version": "3.2.5",
+  "version": "3.3.1",
   "main": "main.js",
   "scripts": {
     "start": "node main.js",

package/src/WebScraper.js

@@ -18,7 +18,8 @@ class WebScraper
 		textOutputPath,
 		csvOutputPath,
 		includeMetadata = false,
-		metadataFields = [] // ['title', 'description', 'author', 'lastModified', etc.]
+		metadataFields = [], // ['title', 'description', 'author', 'lastModified', etc.]
+		userAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:134.0) Gecko/20100101 Firefox/134.0"
 	})
 	{
 		this.baseURL = baseURL;
@@ -30,10 +31,11 @@ class WebScraper
 		this.csvOutputPath = csvOutputPath || path.join( this.scrapResultPath, "train.csv" );
 		this.jsonlOutputPathWithMeta = jsonlOutputPath.replace( ".jsonl", "_with_metadata.jsonl" );
 		this.csvOutputPathWithMeta = csvOutputPath.replace( ".csv", "_with_metadata.csv" );
+		this.userAgent = userAgent;
 		this.includeMetadata = includeMetadata;
 	   this.metadataFields = new Set( metadataFields );
 		this.visited = new Set();
-		this.excludeList = new Set( excludeList );
+		this.excludeList = this.normalizeExcludeList( excludeList );
 		this.exactExcludeList = this.normalizeExcludeList( exactExcludeList );
 		this.allProcessedContent = [];
 	}
@@ -57,7 +59,11 @@ class WebScraper
 		this.visited.add( url );
 		try
 		{
-			const { data, headers } = await axios.get( url );
+			const { data, headers } = await axios.get( url, {
+				headers: {
+					"user-agent": this.userAgent
+				}
+			});
 			const dom = new JSDOM( data, { url });
 			const { document } = dom.window;
 
@@ -334,7 +340,7 @@ class WebScraper
 		};
 	}
 
-	normalizeExcludeList ( list )
+	normalizeExcludeList ( list = [] )
 	{
 		const normalizedSet = new Set();
 		for ( let i = 0; i < list.length; i++ )