clean-web-scraper 3.3.3 → 3.3.5

package/README.md

@@ -35,6 +35,7 @@ npm i clean-web-scraper
 
 git clone https://github.com/mlibre/Clean-Web-Scraper
 cd Clean-Web-Scraper
+sudo pacman -S extra/xorg-server-xvfb chromium
 npm install
 ```
 

package/example-usage.js

@@ -1,5 +1,13 @@
 const WebScraper = require( "./src/WebScraper" );
 
+// const cookies = "cf_clearance=ENHJkpw.ycd1tZ_A.d0O27QdslTN0EHaNurhCznfimg-1738241402-1.2.1.1-BlO.WitkGwE3U3vSamX35xP.AgN1HyvHWL03Jhe.twbn4QWojiw1T4.0M4lE_TcIeZrQ6ErwV9kQBMBKmfU0S6lQth1BJx7UpWn4T6wtFm83LmF.cB13PQYSQgGFGsH7qOkGIjbBhMbceQNp.y2XZgLq_hdntGKSBMe0iCUotx_xsqlzkolQIqnUYID3BLEQXZqNvqJOwkzLZ7.kzrwP42VdEuWEvT4jt7F3TkTaU9rumAp8FSNO1.hnr76Tv23OITm17rPD3__Ghdu1D0E.4v693nEiVYO_KQYNf_8gk0vXP.KAvUKA2zQyBmDXkfW3M1MkoLjFNZCanx9FPRVO7g";
+// const headers = {
+// 	"user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:134.0) Gecko/20100101 Firefox/134.0",
+// 	"Cache-Control": "private",
+// 	"Accept": "application/xml,application/xhtml+xml,text/html;q=0.9, text/plain;q=0.8,image/png,*/*;q=0.5",
+// 	"Cookie": cookies
+// }
+
 
 async function khameneiIrFreePalestineTag ()
 {
@@ -52,9 +60,9 @@ async function decolonizepalestine ()
 
 async function bdsmovement ()
 {
-	// https://bdsmovement.org
+	// https://bdsmovement.net
 	const scraper = new WebScraper({
-		baseURL: "https://bdsmovement.org",
+		baseURL: "https://bdsmovement.net",
 		excludeList: [
 			"https://bdsmovement.net/press-area",
 			"https://bdsmovement.net/privacy-policy",
@@ -68,6 +76,37 @@ async function bdsmovement ()
 		textOutputPath: "./dataset/bdsmovement/texts",
 		csvOutputPath: "./dataset/bdsmovement/train.csv",
 		includeMetadata: true,
+		metadataFields: ["title", "description", "author"],
+		puppeteerProxy: "socks5://127.0.0.1:2080",
+		puppeteerExecutablePath: "/usr/bin/chromium",
+		puppeteerRealProxy: {
+			host: "socks5://127.0.0.1",
+			port: "2080",
+		},
+		usePuppeteer: true
+	});
+	await scraper.start();
+	return scraper;
+}
+
+async function electronicintifada ()
+{
+	// https://electronicintifada.net
+	const scraper = new WebScraper({
+		baseURL: "https://electronicintifada.net",
+		excludeList: [
+			"https://electronicintifada.net/press-area",
+			"https://electronicintifada.net/privacy-policy",
+			"https://electronicintifada.net/get-involved/join-a-bds-campaign",
+			"https://electronicintifada.net/donate_",
+			"https://electronicintifada.net/user",
+			"https://electronicintifada.net/admin"
+		],
+		scrapResultPath: "./dataset/electronicintifada/website",
+		jsonlOutputPath: "./dataset/electronicintifada/train.jsonl",
+		textOutputPath: "./dataset/electronicintifada/texts",
+		csvOutputPath: "./dataset/electronicintifada/train.csv",
+		includeMetadata: true,
 		metadataFields: ["title", "description", "author"]
 	});
 	await scraper.start();
@@ -78,11 +117,11 @@ void async function main ()
 {
 	const khameneiIrFreePalestineTagScraper = await khameneiIrFreePalestineTag();
 	const decolonizepalestineScraper = await decolonizepalestine();
-	// const bdsmovementScraper = await bdsmovement();
+	const bdsmovementScraper = await bdsmovement();
 	await WebScraper.combineResults( "./dataset/combined", [
 		khameneiIrFreePalestineTagScraper,
 		decolonizepalestineScraper,
-		// bdsmovementScraper
+		bdsmovementScraper
 	] );
 
 	// 4

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "clean-web-scraper",
-  "version": "3.3.3",
+  "version": "3.3.5",
   "main": "main.js",
   "scripts": {
     "start": "node main.js",
@@ -26,6 +26,8 @@
     "@mozilla/readability": "^0.5.0",
     "axios": "^1.7.9",
     "eslint": "^9.17.0",
-    "jsdom": "^26.0.0"
+    "jsdom": "^26.0.0",
+    "puppeteer": "^24.1.1",
+    "puppeteer-real-browser": "^1.3.22"
   }
 }

package/src/WebScraper.js

@@ -4,6 +4,8 @@ const { JSDOM } = jsdom;
 const { Readability } = require( "@mozilla/readability" );
 const fs = require( "fs" );
 const path = require( "path" );
+const puppeteer = require( "puppeteer" );
+const { connect } = require( "puppeteer-real-browser" )
 
 class WebScraper
 {
@@ -19,11 +21,11 @@ class WebScraper
 		csvOutputPath,
 		includeMetadata = false,
 		metadataFields = [], // ['title', 'description', 'author', 'lastModified', etc.]
-		headers = {
-			"user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:134.0) Gecko/20100101 Firefox/134.0",
-			"Cache-Control": "private",
-			"Accept": "application/xml,application/xhtml+xml,text/html;q=0.9, text/plain;q=0.8,image/png,*/*;q=0.5"
-		}
+		headers,
+		usePuppeteer,
+		puppeteerProxy, // e.g. http://127.0.0.1:2080
+		puppeteerExecutablePath,
+		puppeteerRealProxy
 	})
 	{
 		this.baseURL = baseURL;
@@ -42,16 +44,67 @@ class WebScraper
 		this.excludeList = this.normalizeExcludeList( excludeList );
 		this.exactExcludeList = this.normalizeExcludeList( exactExcludeList );
 		this.allProcessedContent = [];
+		this.usePuppeteer = usePuppeteer || false;
+		this.puppeteerOptions = {
+			headless: false,
+			userDataDir: "./tmp/browser",
+			defaultViewport: null,
+			args: [
+				"--start-maximized"
+			],
+			"ignoreDefaultArgs": true,
+		}
+		if ( puppeteerProxy )
+		{
+			this.puppeteerOptions.args.push( `--proxy-server=${puppeteerProxy}` );
+		}
+		if ( puppeteerExecutablePath )
+		{
+			this.puppeteerOptions.executablePath = puppeteerExecutablePath;
+		}
+		this.puppeteerRealOptions = {
+			headless: false,
+			args: [],
+			customConfig: {},
+			turnstile: true,
+			connectOption: {},
+			disableXvfb: false,
+			ignoreAllFlags: false,
+			proxy: puppeteerRealProxy
+		}
+		this.puppeteerBrowser = null;
+		this.puppeteerPage = null;
 	}
 
 	async start ()
 	{
-		this.createOutputDirectory();
-		await this.fetchPage( this.startURL, 0 );
-		this.createJSONLFile();
-		this.saveNumberedTextFiles();
-		this.createCSVFile();
-		console.log( "Scraping completed." );
+		try
+		{
+			if ( this.usePuppeteer )
+			{
+				let { browser, page } = await connect( this.puppeteerRealOptions )
+				this.puppeteerBrowser = browser;
+				this.puppeteerPage = page;
+			}
+			this.createOutputDirectory();
+			await this.fetchPage( this.startURL, 0 );
+			this.createJSONLFile();
+			this.saveNumberedTextFiles();
+			this.createCSVFile();
+			console.log( "Scraping completed." );
+		}
+		catch ( error )
+		{
+			console.error( "Error:", error );
+			throw error;
+		}
+		finally
+		{
+			if ( this.puppeteerBrowser )
+			{
+				await this.puppeteerBrowser.close(); // Close the browser after scraping
+			}
+		}
 	}
 
 	async fetchPage ( url, depth )
@@ -63,9 +116,7 @@ class WebScraper
 		this.visited.add( url );
 		try
 		{
-			const { data, headers } = await axios.get( url, {
-				headers: this.headers,
-			});
+			const data = await this.caller( url );
 			const dom = new JSDOM( data, { url });
 			const { document } = dom.window;
 
@@ -78,8 +129,7 @@ class WebScraper
 				{
 					if ( this.isValidContent( article.textContent ) )
 					{
-
-						const metadata = this.metadataextractor( url, document, headers );
+						const metadata = this.metadataextractor( url, document );
 						metadata.depth = depth;
 						this.saveArticle( url, article.textContent, metadata );
 					}
@@ -109,6 +159,82 @@ class WebScraper
 		}
 	}
 
+	async caller ( url )
+	{
+		try
+		{
+			let axiosOptinos = {}
+			if ( this.headers )
+			{
+				axiosOptinos.headers = this.headers
+			}
+			const result = await axios.get( url, axiosOptinos );
+			return result.data
+		}
+		catch ( error )
+		{
+			console.error( `Error fetching ${url}:`, error.message );
+			if ( error.status = 403 && this.usePuppeteer )
+			{
+				// const browser = await puppeteer.launch( this.puppeteerOptions );
+				// const page = await browser.newPage();
+				try
+				{
+					let result;
+					for ( let index = 0; index < 10; index++ )
+					{
+						console.log( `Please solve the CAPTCHA on the opened browser window for ${url}` );
+						result = await this.goToUrl( url ) ;
+						if ( this.isValidContent( result.htmlContent ) )
+						{
+							break
+						}
+					}
+					return result.htmlContent;
+				}
+				catch ( error )
+				{
+					console.error( `Error solving CAPTCHA for ${url}:`, error.message, error );
+					throw error;
+				}
+
+			}
+			throw error;
+		}
+	}
+
+	async goToUrl ( url )
+	{
+		let pages = await this.puppeteerBrowser.pages();
+		let page = pages[0];
+		page.setDefaultNavigationTimeout( 10000 );
+		await page.goto( url );
+		pages = await this.puppeteerBrowser.pages();
+		page = pages[0];
+		page.setDefaultNavigationTimeout( 10000 );
+		await this.waitForPageToLoad( page );
+		pages = await this.puppeteerBrowser.pages();
+		page = pages[0];
+		page.setDefaultNavigationTimeout( 10000 );
+		if ( page )
+		{
+			let htmlContent = await page.content();
+			return { pages, page, htmlContent };
+		}
+	}
+
+	async waitForPageToLoad ( page )
+	{
+		try
+		{
+			await page.waitForNavigation({ waitUntil: "networkidle0" });
+		}
+		catch ( error )
+		{
+			console.log( error );
+		}
+	}
+
 	extractLinks ( data )
 	{
 		const links = new Set();
@@ -329,7 +455,7 @@ class WebScraper
 		return filteredMetadata;
 	}
 
-	metadataextractor ( url, document, headers )
+	metadataextractor ( url, document )
 	{
 		return {
 			url,
@@ -337,9 +463,6 @@ class WebScraper
 			description: document.querySelector( "meta[name=\"description\"]" )?.content,
 			keywords: document.querySelector( "meta[name=\"keywords\"]" )?.content,
 			author: document.querySelector( "meta[name=\"author\"]" )?.content,
-			lastModified: headers["last-modified"],
-			contentType: headers["content-type"],
-			contentLength: headers["content-length"],
 			language: document.documentElement.lang || document.querySelector( "html" )?.getAttribute( "lang" ),
 			canonicalUrl: document.querySelector( "link[rel=\"canonical\"]" )?.href,
 			ogTitle: document.querySelector( "meta[property=\"og:title\"]" )?.content,
@@ -416,16 +539,19 @@ class WebScraper
 		// List of phrases that indicate invalid content
 		const invalidPhrases = [
 			"verifying that you are not a robot",
+			"verifying you are human. this may take a few seconds.",
+			"verify you are human by completing the action below",
 			"checking if the site connection is secure",
 			"please wait while we verify",
 			"please enable javascript",
 			"access denied",
+			"verifying you are human",
 			"captcha verification"
 		];
 
 		const hasInvalidPhrases = invalidPhrases.some( phrase => { return cleanContent.includes( phrase ) });
 		// Check content length
-		if ( cleanContent.length < 100 && hasInvalidPhrases )
+		if ( cleanContent.length < 100 || hasInvalidPhrases )
 		{
 			return false;
 		}