clean-web-scraper 3.3.4 → 3.3.6

package/example-usage.js

@@ -82,7 +82,8 @@ async function bdsmovement ()
 		puppeteerRealProxy: {
 			host: "socks5://127.0.0.1",
 			port: "2080",
-		}
+		},
+		// usePuppeteer: true
 	});
 	await scraper.start();
 	return scraper;
@@ -94,12 +95,19 @@ async function electronicintifada ()
 	const scraper = new WebScraper({
 		baseURL: "https://electronicintifada.net",
 		excludeList: [
-			"https://electronicintifada.net/press-area",
-			"https://electronicintifada.net/privacy-policy",
-			"https://electronicintifada.net/get-involved/join-a-bds-campaign",
-			"https://electronicintifada.net/donate_",
-			"https://electronicintifada.net/user",
-			"https://electronicintifada.net/admin"
+			"https://electronicintifada.net/updates",
+			"https://electronicintifada.net/taxonomy/term/",
+			"https://electronicintifada.net/tags/",
+			"https://electronicintifada.net/blog",
+			"https://electronicintifada.net/people",
+			"https://electronicintifada.net/location"
+		],
+		exactExcludeList: [
+			"https://electronicintifada.net",
+			"https://electronicintifada.net/blog",
+			"https://electronicintifada.net/news",
+			"https://electronicintifada.net/opinion",
+			"https://electronicintifada.net/review",
 		],
 		scrapResultPath: "./dataset/electronicintifada/website",
 		jsonlOutputPath: "./dataset/electronicintifada/train.jsonl",
@@ -114,13 +122,15 @@ async function electronicintifada ()
 
 void async function main ()
 {
-	// const khameneiIrFreePalestineTagScraper = await khameneiIrFreePalestineTag();
+	const khameneiIrFreePalestineTagScraper = await khameneiIrFreePalestineTag();
 	const decolonizepalestineScraper = await decolonizepalestine();
-	// const bdsmovementScraper = await bdsmovement();
+	const bdsmovementScraper = await bdsmovement();
+	const electronicintifadaScraper = await electronicintifada();
 	await WebScraper.combineResults( "./dataset/combined", [
-		// khameneiIrFreePalestineTagScraper,
+		khameneiIrFreePalestineTagScraper,
 		decolonizepalestineScraper,
-		// bdsmovementScraper
+		bdsmovementScraper,
+		electronicintifadaScraper
 	] );
 
 	// 4

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "clean-web-scraper",
-  "version": "3.3.4",
+  "version": "3.3.6",
   "main": "main.js",
   "scripts": {
     "start": "node main.js",

package/src/WebScraper.js

@@ -22,6 +22,7 @@ class WebScraper
 		includeMetadata = false,
 		metadataFields = [], // ['title', 'description', 'author', 'lastModified', etc.]
 		headers,
+		usePuppeteer,
 		puppeteerProxy, // e.g. http://127.0.0.1:2080
 		puppeteerExecutablePath,
 		puppeteerRealProxy
@@ -43,7 +44,7 @@ class WebScraper
 		this.excludeList = this.normalizeExcludeList( excludeList );
 		this.exactExcludeList = this.normalizeExcludeList( exactExcludeList );
 		this.allProcessedContent = [];
-		this.usePuppeteer = false;
+		this.usePuppeteer = usePuppeteer || false;
 		this.puppeteerOptions = {
 			headless: false,
 			userDataDir: "./tmp/browser",
@@ -71,16 +72,39 @@ class WebScraper
 			ignoreAllFlags: false,
 			proxy: puppeteerRealProxy
 		}
+		this.puppeteerBrowser = null;
+		this.puppeteerPage = null;
 	}
 
 	async start ()
 	{
-		this.createOutputDirectory();
-		await this.fetchPage( this.startURL, 0 );
-		this.createJSONLFile();
-		this.saveNumberedTextFiles();
-		this.createCSVFile();
-		console.log( "Scraping completed." );
+		try
+		{
+			if ( this.usePuppeteer )
+			{
+				let { browser, page } = await connect( this.puppeteerRealOptions )
+				this.puppeteerBrowser = browser;
+				this.puppeteerPage = page;
+			}
+			this.createOutputDirectory();
+			await this.fetchPage( this.startURL, 0 );
+			this.createJSONLFile();
+			this.saveNumberedTextFiles();
+			this.createCSVFile();
+			console.log( "Scraping completed." );
+		}
+		catch ( error )
+		{
+			console.error( "Error:", error );
+			throw error;
+		}
+		finally
+		{
+			if ( this.puppeteerBrowser )
+			{
+				await this.puppeteerBrowser.close(); // Close the browser after scraping
+			}
+		}
 	}
 
 	async fetchPage ( url, depth )
@@ -152,52 +176,53 @@ class WebScraper
 			console.error( `Error fetching ${url}:`, error.message );
 			if ( error.status = 403 && this.usePuppeteer )
 			{
-				let { browser, page } = await connect( this.puppeteerRealOptions )
-
 				// const browser = await puppeteer.launch( this.puppeteerOptions );
 				// const page = await browser.newPage();
 				try
 				{
-					let htmlContent;
+					let result;
 					for ( let index = 0; index < 10; index++ )
 					{
-						const pages = await browser.pages();
-						page = pages[0];
-						page.setDefaultNavigationTimeout( 30000 )
-						await page.goto( url );
 						console.log( `Please solve the CAPTCHA on the opened browser window for ${url}` );
-						await this.waitForPageToLoad( page );
-						htmlContent = await page.content();
-						if ( this.isValidContent( htmlContent ) )
+						result = await this.goToUrl( url ) ;
+						if ( this.isValidContent( result.htmlContent ) )
 						{
 							break
 						}
-						page = pages[0];
-						page.setDefaultNavigationTimeout( 30000 )
-						await this.waitForPageToLoad( page );
-						htmlContent = await page.content();
-						if ( this.isValidContent( htmlContent ) )
-						{
-							break
-						}
-						await page.goto( url );
 					}
-					return htmlContent;
+					return result.htmlContent;
 				}
 				catch ( error )
 				{
 					console.error( `Error solving CAPTCHA for ${url}:`, error.message, error );
 					throw error;
 				}
-				finally
-				{
-					await browser.close(); // Close the browser after scraping
-				}
+
 			}
 			throw error;
 		}
 	}
 
+	async goToUrl ( url )
+	{
+		let pages = await this.puppeteerBrowser.pages();
+		let page = pages[0];
+		page.setDefaultNavigationTimeout( 10000 );
+		await page.goto( url );
+		pages = await this.puppeteerBrowser.pages();
+		page = pages[0];
+		page.setDefaultNavigationTimeout( 10000 );
+		await this.waitForPageToLoad( page );
+		pages = await this.puppeteerBrowser.pages();
+		page = pages[0];
+		page.setDefaultNavigationTimeout( 10000 );
+		if ( page )
+		{
+			let htmlContent = await page.content();
+			return { pages, page, htmlContent };
+		}
+	}
+
 	async waitForPageToLoad ( page )
 	{
 		try
@@ -412,6 +437,23 @@ class WebScraper
 		// processed = processed.replace(/\[.*?\]/g, ''); // Removes all content within square brackets
 		// processed = processed.replace(/\(.*?\)/g, ''); // Removes all content within parentheses
 
+		// Remove specified words from the end of content, handling multiple occurrences
+		const wordsToTrim = ["Facebook", "Twitter", "Donate Now", "Instagram"];
+		let changed = true;
+
+		while ( changed )
+		{
+			changed = false;
+			for ( let i = 0; i < wordsToTrim.length; i++ )
+			{
+				const oldProcessed = processed;
+				processed = processed.replace( new RegExp( `\\s*${wordsToTrim[i]}\\s*$`, "g" ), "" ).trim();
+				if ( oldProcessed !== processed )
+				{
+					changed = true;
+				}
+			}
+		}
 		return processed;
 	}