clean-web-scraper 3.3.2 → 3.3.4

package/README.md

@@ -35,6 +35,7 @@ npm i clean-web-scraper
 
 git clone https://github.com/mlibre/Clean-Web-Scraper
 cd Clean-Web-Scraper
+sudo pacman -S extra/xorg-server-xvfb chromium
 npm install
 ```
 

package/example-usage.js

@@ -1,5 +1,13 @@
 const WebScraper = require( "./src/WebScraper" );
 
+// const cookies = "cf_clearance=ENHJkpw.ycd1tZ_A.d0O27QdslTN0EHaNurhCznfimg-1738241402-1.2.1.1-BlO.WitkGwE3U3vSamX35xP.AgN1HyvHWL03Jhe.twbn4QWojiw1T4.0M4lE_TcIeZrQ6ErwV9kQBMBKmfU0S6lQth1BJx7UpWn4T6wtFm83LmF.cB13PQYSQgGFGsH7qOkGIjbBhMbceQNp.y2XZgLq_hdntGKSBMe0iCUotx_xsqlzkolQIqnUYID3BLEQXZqNvqJOwkzLZ7.kzrwP42VdEuWEvT4jt7F3TkTaU9rumAp8FSNO1.hnr76Tv23OITm17rPD3__Ghdu1D0E.4v693nEiVYO_KQYNf_8gk0vXP.KAvUKA2zQyBmDXkfW3M1MkoLjFNZCanx9FPRVO7g";
+// const headers = {
+// 	"user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:134.0) Gecko/20100101 Firefox/134.0",
+// 	"Cache-Control": "private",
+// 	"Accept": "application/xml,application/xhtml+xml,text/html;q=0.9, text/plain;q=0.8,image/png,*/*;q=0.5",
+// 	"Cookie": cookies
+// }
+
 
 async function khameneiIrFreePalestineTag ()
 {
@@ -52,9 +60,9 @@ async function decolonizepalestine ()
 
 async function bdsmovement ()
 {
-	// https://bdsmovement.org
+	// https://bdsmovement.net
 	const scraper = new WebScraper({
-		baseURL: "https://bdsmovement.org",
+		baseURL: "https://bdsmovement.net",
 		excludeList: [
 			"https://bdsmovement.net/press-area",
 			"https://bdsmovement.net/privacy-policy",
@@ -68,6 +76,36 @@ async function bdsmovement ()
 		textOutputPath: "./dataset/bdsmovement/texts",
 		csvOutputPath: "./dataset/bdsmovement/train.csv",
 		includeMetadata: true,
+		metadataFields: ["title", "description", "author"],
+		puppeteerProxy: "socks5://127.0.0.1:2080",
+		puppeteerExecutablePath: "/usr/bin/chromium",
+		puppeteerRealProxy: {
+			host: "socks5://127.0.0.1",
+			port: "2080",
+		}
+	});
+	await scraper.start();
+	return scraper;
+}
+
+async function electronicintifada ()
+{
+	// https://electronicintifada.net
+	const scraper = new WebScraper({
+		baseURL: "https://electronicintifada.net",
+		excludeList: [
+			"https://electronicintifada.net/press-area",
+			"https://electronicintifada.net/privacy-policy",
+			"https://electronicintifada.net/get-involved/join-a-bds-campaign",
+			"https://electronicintifada.net/donate_",
+			"https://electronicintifada.net/user",
+			"https://electronicintifada.net/admin"
+		],
+		scrapResultPath: "./dataset/electronicintifada/website",
+		jsonlOutputPath: "./dataset/electronicintifada/train.jsonl",
+		textOutputPath: "./dataset/electronicintifada/texts",
+		csvOutputPath: "./dataset/electronicintifada/train.csv",
+		includeMetadata: true,
 		metadataFields: ["title", "description", "author"]
 	});
 	await scraper.start();
@@ -77,13 +115,13 @@ async function bdsmovement ()
 void async function main ()
 {
 	// const khameneiIrFreePalestineTagScraper = await khameneiIrFreePalestineTag();
-	// const decolonizepalestineScraper = await decolonizepalestine();
-	const bdsmovementScraper = await bdsmovement();
-	// await WebScraper.combineResults( "./dataset/combined", [
-	// 	khameneiIrFreePalestineTagScraper,
-	// 	decolonizepalestineScraper,
-	// 	bdsmovementScraper
-	// ] );
+	const decolonizepalestineScraper = await decolonizepalestine();
+	// const bdsmovementScraper = await bdsmovement();
+	await WebScraper.combineResults( "./dataset/combined", [
+		// khameneiIrFreePalestineTagScraper,
+		decolonizepalestineScraper,
+		// bdsmovementScraper
+	] );
 
 	// 4
 	// https://electronicintifada.net/

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "clean-web-scraper",
-  "version": "3.3.2",
+  "version": "3.3.4",
   "main": "main.js",
   "scripts": {
     "start": "node main.js",
@@ -26,6 +26,8 @@
     "@mozilla/readability": "^0.5.0",
     "axios": "^1.7.9",
     "eslint": "^9.17.0",
-    "jsdom": "^26.0.0"
+    "jsdom": "^26.0.0",
+    "puppeteer": "^24.1.1",
+    "puppeteer-real-browser": "^1.3.22"
   }
 }

package/src/WebScraper.js

@@ -4,6 +4,8 @@ const { JSDOM } = jsdom;
 const { Readability } = require( "@mozilla/readability" );
 const fs = require( "fs" );
 const path = require( "path" );
+const puppeteer = require( "puppeteer" );
+const { connect } = require( "puppeteer-real-browser" )
 
 class WebScraper
 {
@@ -19,11 +21,10 @@ class WebScraper
 		csvOutputPath,
 		includeMetadata = false,
 		metadataFields = [], // ['title', 'description', 'author', 'lastModified', etc.]
-		headers = {
-			"user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:134.0) Gecko/20100101 Firefox/134.0",
-			"Cache-Control": "private",
-			"Accept": "application/xml,application/xhtml+xml,text/html;q=0.9, text/plain;q=0.8,image/png,*/*;q=0.5"
-		}
+		headers,
+		puppeteerProxy, // e.g. http://127.0.0.1:2080
+		puppeteerExecutablePath,
+		puppeteerRealProxy
 	})
 	{
 		this.baseURL = baseURL;
@@ -42,6 +43,34 @@ class WebScraper
 		this.excludeList = this.normalizeExcludeList( excludeList );
 		this.exactExcludeList = this.normalizeExcludeList( exactExcludeList );
 		this.allProcessedContent = [];
+		this.usePuppeteer = false;
+		this.puppeteerOptions = {
+			headless: false,
+			userDataDir: "./tmp/browser",
+			defaultViewport: null,
+			args: [
+				"--start-maximized"
+			],
+			"ignoreDefaultArgs": true,
+		}
+		if ( puppeteerProxy )
+		{
+			this.puppeteerOptions.args.push( `--proxy-server=${puppeteerProxy}` );
+		}
+		if ( puppeteerExecutablePath )
+		{
+			this.puppeteerOptions.executablePath = puppeteerExecutablePath;
+		}
+		this.puppeteerRealOptions = {
+			headless: false,
+			args: [],
+			customConfig: {},
+			turnstile: true,
+			connectOption: {},
+			disableXvfb: false,
+			ignoreAllFlags: false,
+			proxy: puppeteerRealProxy
+		}
 	}
 
 	async start ()
@@ -63,9 +92,7 @@ class WebScraper
 		this.visited.add( url );
 		try
 		{
-			const { data, headers } = await axios.get( url, {
-				headers: this.headers,
-			});
+			const data = await this.caller( url );
 			const dom = new JSDOM( data, { url });
 			const { document } = dom.window;
 
@@ -78,8 +105,7 @@ class WebScraper
 				{
 					if ( this.isValidContent( article.textContent ) )
 					{
-
-						const metadata = this.metadataextractor( url, document, headers );
+						const metadata = this.metadataextractor( url, document );
 						metadata.depth = depth;
 						this.saveArticle( url, article.textContent, metadata );
 					}
@@ -109,6 +135,81 @@ class WebScraper
 		}
 	}
 
+	async caller ( url )
+	{
+		try
+		{
+			let axiosOptinos = {}
+			if ( this.headers )
+			{
+				axiosOptinos.headers = this.headers
+			}
+			const result = await axios.get( url, axiosOptinos );
+			return result.data
+		}
+		catch ( error )
+		{
+			console.error( `Error fetching ${url}:`, error.message );
+			if ( error.status = 403 && this.usePuppeteer )
+			{
+				let { browser, page } = await connect( this.puppeteerRealOptions )
+
+				// const browser = await puppeteer.launch( this.puppeteerOptions );
+				// const page = await browser.newPage();
+				try
+				{
+					let htmlContent;
+					for ( let index = 0; index < 10; index++ )
+					{
+						const pages = await browser.pages();
+						page = pages[0];
+						page.setDefaultNavigationTimeout( 30000 )
+						await page.goto( url );
+						console.log( `Please solve the CAPTCHA on the opened browser window for ${url}` );
+						await this.waitForPageToLoad( page );
+						htmlContent = await page.content();
+						if ( this.isValidContent( htmlContent ) )
+						{
+							break
+						}
+						page = pages[0];
+						page.setDefaultNavigationTimeout( 30000 )
+						await this.waitForPageToLoad( page );
+						htmlContent = await page.content();
+						if ( this.isValidContent( htmlContent ) )
+						{
+							break
+						}
+						await page.goto( url );
+					}
+					return htmlContent;
+				}
+				catch ( error )
+				{
+					console.error( `Error solving CAPTCHA for ${url}:`, error.message, error );
+					throw error;
+				}
+				finally
+				{
+					await browser.close(); // Close the browser after scraping
+				}
+			}
+			throw error;
+		}
+	}
+
+	async waitForPageToLoad ( page )
+	{
+		try
+		{
+			await page.waitForNavigation({ waitUntil: "networkidle0" });
+		}
+		catch ( error )
+		{
+			console.log( error );
+		}
+	}
+
 	extractLinks ( data )
 	{
 		const links = new Set();
@@ -329,7 +430,7 @@ class WebScraper
 		return filteredMetadata;
 	}
 
-	metadataextractor ( url, document, headers )
+	metadataextractor ( url, document )
 	{
 		return {
 			url,
@@ -337,9 +438,6 @@ class WebScraper
 			description: document.querySelector( "meta[name=\"description\"]" )?.content,
 			keywords: document.querySelector( "meta[name=\"keywords\"]" )?.content,
 			author: document.querySelector( "meta[name=\"author\"]" )?.content,
-			lastModified: headers["last-modified"],
-			contentType: headers["content-type"],
-			contentLength: headers["content-length"],
 			language: document.documentElement.lang || document.querySelector( "html" )?.getAttribute( "lang" ),
 			canonicalUrl: document.querySelector( "link[rel=\"canonical\"]" )?.href,
 			ogTitle: document.querySelector( "meta[property=\"og:title\"]" )?.content,
@@ -408,6 +506,33 @@ class WebScraper
 		fs.mkdirSync( path.join( __dirname, this.textOutputPath ), { recursive: true });
 	}
 
+	isValidContent ( content )
+	{
+		// Remove whitespace and newlines for checking
+		const cleanContent = content.replace( /\s+/g, " " ).trim().toLowerCase();
+
+		// List of phrases that indicate invalid content
+		const invalidPhrases = [
+			"verifying that you are not a robot",
+			"verifying you are human. this may take a few seconds.",
+			"verify you are human by completing the action below",
+			"checking if the site connection is secure",
+			"please wait while we verify",
+			"please enable javascript",
+			"access denied",
+			"verifying you are human",
+			"captcha verification"
+		];
+
+		const hasInvalidPhrases = invalidPhrases.some( phrase => { return cleanContent.includes( phrase ) });
+		// Check content length
+		if ( cleanContent.length < 100 || hasInvalidPhrases )
+		{
+			return false;
+		}
+		return true;
+	}
+
 	static sleep ( ms )
 	{
 		return new Promise( resolve => { return setTimeout( resolve, ms ) });
@@ -448,12 +573,18 @@ class WebScraper
 		for ( const website of websites )
 		{
 			const jsonlContent = fs.readFileSync( path.join( __dirname, website.jsonlOutputPath ), "utf-8" );
-			jsonlOutput.write( jsonlContent );
+			if ( jsonlContent )
+			{
+				jsonlOutput.write( jsonlContent );
+			}
 
 			if ( website.includeMetadata )
 			{
 				const jsonlMetaContent = fs.readFileSync( path.join( __dirname, website.jsonlOutputPathWithMeta ), "utf-8" );
-				jsonlMetaOutput.write( jsonlMetaContent );
+				if ( jsonlMetaContent )
+				{
+					jsonlMetaOutput.write( jsonlMetaContent );
+				}
 			}
 		}
 
@@ -480,7 +611,10 @@ class WebScraper
 			.split( "\n" )
 			.slice( 1 )
 			.filter( line => { return line.trim() });
-			csvOutput.write( `${csvContent.join( "\n" )}\n` );
+			if ( csvContent.length > 0 )
+			{
+				csvOutput.write( `${csvContent.join( "\n" )}\n` );
+			}
 
 			if ( website.includeMetadata )
 			{
@@ -488,7 +622,10 @@ class WebScraper
 				.split( "\n" )
 				.slice( 1 )
 				.filter( line => { return line.trim() });
-				csvMetaOutput.write( `${csvMetaContent.join( "\n" )}\n` );
+				if ( csvMetaContent.length > 0 )
+				{
+					csvMetaOutput.write( `${csvMetaContent.join( "\n" )}\n` );
+				}
 			}
 		}
 
@@ -496,30 +633,6 @@ class WebScraper
 		csvMetaOutput.end();
 	}
 
-	isValidContent ( content )
-	{
-		// Remove whitespace and newlines for checking
-		const cleanContent = content.replace( /\s+/g, " " ).trim().toLowerCase();
-
-		// List of phrases that indicate invalid content
-		const invalidPhrases = [
-			"verifying that you are not a robot",
-			"checking if the site connection is secure",
-			"please wait while we verify",
-			"please enable javascript",
-			"access denied",
-			"captcha verification"
-		];
-
-		const hasInvalidPhrases = invalidPhrases.some( phrase => { return cleanContent.includes( phrase ) });
-		// Check content length
-		if ( cleanContent.length < 100 && hasInvalidPhrases )
-		{
-			return false;
-		}
-		return true;
-	}
-
 	static combineTextFiles ( fullOutputPath, websites )
 	{
 		let textFileCounter = 1;
@@ -553,7 +666,6 @@ class WebScraper
 			}
 		}
 	}
-
 }
 
 module.exports = WebScraper;