clean-web-scraper 3.8.1 → 3.8.3

package/example-usage.js

@@ -29,7 +29,8 @@ async function palianswers ( enable )
 		textOutputPath: "./dataset/palianswers/texts",
 		csvOutputPath: "./dataset/palianswers/train.csv",
 		includeMetadata: true,
-		metadataFields: ["author", "title", "description", "dateScrapedDate"]
+		metadataFields: ["author", "title", "description", "dateScrapedDate"],
+		retryDelay: 10000
 	});
 	if ( enable )
 	{
@@ -54,7 +55,8 @@ async function khameneiIrFreePalestineTag ( enable )
 		textOutputPath: "./dataset/khamenei-ir-free-palestine-tag/texts",
 		csvOutputPath: "./dataset/khamenei-ir-free-palestine-tag/train.csv",
 		includeMetadata: true,
-		metadataFields: ["author", "title", "description", "dateScrapedDate"]
+		metadataFields: ["author", "title", "description", "dateScrapedDate"],
+		retryDelay: 10000
 	});
 	if ( enable )
 	{
@@ -84,40 +86,8 @@ async function decolonizepalestine ( enable )
 		textOutputPath: "./dataset/decolonizepalestine/texts",
 		csvOutputPath: "./dataset/decolonizepalestine/train.csv",
 		includeMetadata: true,
-		metadataFields: ["author", "title", "description", "dateScrapedDate"]
-	});
-	if ( enable )
-	{
-		await scraper.start();
-	}
-	return scraper;
-}
-
-async function bdsmovement ( enable )
-{
-	// https://bdsmovement.net
-	const scraper = new WebScraper({
-		baseURL: "https://bdsmovement.net",
-		excludeList: [
-			"https://bdsmovement.net/press-area",
-			"https://bdsmovement.net/privacy-policy",
-			"https://bdsmovement.net/get-involved/join-a-bds-campaign",
-			"https://bdsmovement.net/donate_",
-			"https://bdsmovement.net/user",
-			"https://bdsmovement.net/admin"
-		],
-		scrapResultPath: "./dataset/bdsmovement/website",
-		jsonlOutputPath: "./dataset/bdsmovement/train.jsonl",
-		textOutputPath: "./dataset/bdsmovement/texts",
-		csvOutputPath: "./dataset/bdsmovement/train.csv",
-		includeMetadata: true,
 		metadataFields: ["author", "title", "description", "dateScrapedDate"],
-		puppeteerProxy: "socks5://127.0.0.1:2080",
-		puppeteerExecutablePath: "/usr/bin/chromium",
-		puppeteerRealProxy: {
-			host: "socks5://127.0.0.1",
-			port: "2080",
-		},
+		retryDelay: 10000
 	});
 	if ( enable )
 	{
@@ -155,53 +125,12 @@ async function electronicintifada ( enable )
 		textOutputPath: "./dataset/electronicintifada/texts",
 		csvOutputPath: "./dataset/electronicintifada/train.csv",
 		includeMetadata: true,
+		metadataFields: ["author", "title", "description", "dateScrapedDate"],
+		maxDepth: 10,
 		maxArticles: 2000,
+		concurrencyLimit: 4,
 		axiosHeaders: headers,
-		metadataFields: ["author", "title", "description", "dateScrapedDate"]
-	});
-	if ( enable )
-	{
-		await scraper.start();
-	}
-	return scraper;
-}
-
-async function palestineremembered ( enable )
-{
-	// https://www.palestineremembered.com
-	const scraper = new WebScraper({
-		baseURL: "https://www.palestineremembered.com",
-		startURL: "https://www.palestineremembered.com/ZionistFAQ.html",
-		excludeList: [
-			"https://www.palestineremembered.com/GeoPoints",
-			"https://www.palestineremembered.com/Donate",
-			"https://www.palestineremembered.com/ContactUs.html",
-			"https://www.palestineremembered.com/tags/Looting-Palestinian-properties.html",
-			"https://www.palestineremembered.com/ar/",
-			"https://www.palestineremembered.com/OldNewPictures.html",
-			"https://www.palestineremembered.com/Maps/index.html",
-			"https://www.palestineremembered.com/OralHistory/Interviews-Listing/",
-			"https://www.palestineremembered.com/Acre/Famous-Zionist-Quotes/Story637.html",
-			"https://www.palestineremembered.com/Articles/General/Story2045.html",
-			"https://www.palestineremembered.com/AllTownsListing.html",
-			"https://www.palestineremembered.com/Articles/General/ar/",
-			"https://www.palestineremembered.com/SiteVideos.html"
-		],
-		exactExcludeList: [
-			"https://www.palestineremembered.com/index.html",
-			"https://www.palestineremembered.com/ZionistFAQ.html"
-		],
-		scrapResultPath: "./dataset/palestineremembered/website",
-		jsonlOutputPath: "./dataset/palestineremembered/train.jsonl",
-		textOutputPath: "./dataset/palestineremembered/texts",
-		csvOutputPath: "./dataset/palestineremembered/train.csv",
-		includeMetadata: true,
-		metadataFields: ["author", "title", "description", "dateScrapedDate"],
-		axiosProxy: {
-			host: "localhost",
-			port: 2080,
-			protocol: "http"
-		}
+		retryDelay: 10000
 	});
 	if ( enable )
 	{
@@ -259,15 +188,18 @@ async function mondoweiss ( enable )
 		jsonlOutputPath: "./dataset/mondoweiss/train.jsonl",
 		textOutputPath: "./dataset/mondoweiss/texts",
 		csvOutputPath: "./dataset/mondoweiss/train.csv",
-		includeMetadata: true,
 		maxArticles: 2500,
 		maxRetries: 2,
+		concurrencyLimit: 4,
 		axiosHeaders: headers,
 		axiosProxy: {
 			host: "localhost",
 			port: 2080,
 			protocol: "http"
 		},
+		maxDepth: 10,
+		retryDelay: 10000,
+		includeMetadata: true,
 		metadataFields: ["author", "title", "description", "dateScrapedDate"]
 	});
 	if ( enable )
@@ -277,14 +209,90 @@ async function mondoweiss ( enable )
 	return scraper;
 }
 
+async function bdsmovement ( enable )
+{
+	// https://bdsmovement.net
+	const scraper = new WebScraper({
+		baseURL: "https://bdsmovement.net",
+		excludeList: [
+			"https://bdsmovement.net/press-area",
+			"https://bdsmovement.net/privacy-policy",
+			"https://bdsmovement.net/get-involved/join-a-bds-campaign",
+			"https://bdsmovement.net/donate_",
+			"https://bdsmovement.net/user",
+			"https://bdsmovement.net/admin"
+		],
+		scrapResultPath: "./dataset/bdsmovement/website",
+		jsonlOutputPath: "./dataset/bdsmovement/train.jsonl",
+		textOutputPath: "./dataset/bdsmovement/texts",
+		csvOutputPath: "./dataset/bdsmovement/train.csv",
+		includeMetadata: true,
+		metadataFields: ["author", "title", "description", "dateScrapedDate"],
+		puppeteerProxy: "socks5://127.0.0.1:2080",
+		puppeteerExecutablePath: "/usr/bin/chromium",
+		puppeteerRealProxy: {
+			host: "socks5://127.0.0.1",
+			port: "2080",
+		},
+	});
+	if ( enable )
+	{
+		await scraper.start();
+	}
+	return scraper;
+}
+
+async function palestineremembered ( enable )
+{
+	// https://www.palestineremembered.com
+	const scraper = new WebScraper({
+		baseURL: "https://www.palestineremembered.com",
+		startURL: "https://www.palestineremembered.com/ZionistFAQ.html",
+		excludeList: [
+			"https://www.palestineremembered.com/GeoPoints",
+			"https://www.palestineremembered.com/Donate",
+			"https://www.palestineremembered.com/ContactUs.html",
+			"https://www.palestineremembered.com/tags/Looting-Palestinian-properties.html",
+			"https://www.palestineremembered.com/ar/",
+			"https://www.palestineremembered.com/OldNewPictures.html",
+			"https://www.palestineremembered.com/Maps/index.html",
+			"https://www.palestineremembered.com/OralHistory/Interviews-Listing/",
+			"https://www.palestineremembered.com/Acre/Famous-Zionist-Quotes/Story637.html",
+			"https://www.palestineremembered.com/Articles/General/Story2045.html",
+			"https://www.palestineremembered.com/AllTownsListing.html",
+			"https://www.palestineremembered.com/Articles/General/ar/",
+			"https://www.palestineremembered.com/SiteVideos.html"
+		],
+		exactExcludeList: [
+			"https://www.palestineremembered.com/index.html",
+			"https://www.palestineremembered.com/ZionistFAQ.html"
+		],
+		scrapResultPath: "./dataset/palestineremembered/website",
+		jsonlOutputPath: "./dataset/palestineremembered/train.jsonl",
+		textOutputPath: "./dataset/palestineremembered/texts",
+		csvOutputPath: "./dataset/palestineremembered/train.csv",
+		includeMetadata: true,
+		metadataFields: ["author", "title", "description", "dateScrapedDate"],
+		axiosProxy: {
+			host: "localhost",
+			port: 2080,
+			protocol: "http"
+		}
+	});
+	if ( enable )
+	{
+		await scraper.start();
+	}
+	return scraper;
+}
 
 void async function main ()
 {
-	const palianswersScraper = await palianswers( false );
-	const decolonizepalestineScraper = await decolonizepalestine( false );
-	const khameneiIrFreePalestineTagScraper = await khameneiIrFreePalestineTag( false );
-	const electronicintifadaScraper = await electronicintifada( false );
-	const standWithPalestineScraper = await standWithPalestine( false );
+	const palianswersScraper = await palianswers( true );
+	const decolonizepalestineScraper = await decolonizepalestine( true );
+	const khameneiIrFreePalestineTagScraper = await khameneiIrFreePalestineTag( true );
+	const electronicintifadaScraper = await electronicintifada( true );
+	const standWithPalestineScraper = await standWithPalestine( true );
 	const mondoweisScraper = await mondoweiss( true );
 	const bdsmovementScraper = await bdsmovement( false );
 	const palestinerememberedScraper = await palestineremembered( false );

package/main.js

@@ -16,6 +16,7 @@ class WebScraper
 		maxArticles,
 		concurrencyLimit,
 		maxRetries,
+		retryDelay,
 
 		// URL filtering
 		excludeList = [],
@@ -53,6 +54,7 @@ class WebScraper
 		this.maxArticles = maxArticles || Infinity;
 		this.concurrencyLimit = concurrencyLimit || 2;
 		this.maxRetries = maxRetries || 5;
+		this.retryDelay = retryDelay || 40000;
 
 		// Output paths setup
 		this.scrapResultPath = scrapResultPath;
@@ -357,6 +359,10 @@ class WebScraper
 		{
 			urlPath = "/index";
 		}
+		else if ( urlPath.endsWith( "/" ) )
+		{
+			urlPath = urlPath.slice( 0, -1 );
+		}
 		const filePath = path.join( __dirname, this.scrapResultPath, urlPath );
 		const dir = path.dirname( filePath );
 
@@ -579,14 +585,14 @@ class WebScraper
 			{
 				if ( this.hasReachedMax( ) )
 				{
-					throw new Error( "Max reached" );
+					break;
 				}
 				return await axios.get( url, options );
 			}
 			catch ( error )
 			{
 				if ( attempt >= this.maxRetries ) throw error;
-				await WebScraper.sleep( 40000 * attempt );
+				await WebScraper.sleep( this.retryDelay * attempt );
 				console.error( `Retrying request to ${url} (Attempt ${attempt + 1}/${this.maxRetries})`, error.message, error.code );
 			}
 		}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "clean-web-scraper",
-  "version": "3.8.1",
+  "version": "3.8.3",
   "main": "main.js",
   "scripts": {
     "start": "node main.js",