clean-web-scraper 3.5.1 → 3.5.3

package/example-usage.js

@@ -8,8 +8,36 @@ const WebScraper = require( "./src/WebScraper" );
 // 	"Cookie": cookies
 // }
 
+async function palianswers ( enable )
+{
+	const scraper = new WebScraper({
+		baseURL: "https://palianswers.com",
+		excludeList: [
+			"https://palianswers.com/chat/",
+			"https://palianswers.com/become-a-volunteer/",
+			"https://palianswers.com/other-resources/",
+			"https://palianswers.com/request-a-rebuttal/",
+			"https://palianswers.com/submit-a-rebuttal/",
+			"https://palianswers.com/themes/"
+		],
+		exactExcludeList: [
+			"https://palianswers.com/",
+		],
+		scrapResultPath: "./dataset/palianswers/website",
+		jsonlOutputPath: "./dataset/palianswers/train.jsonl",
+		textOutputPath: "./dataset/palianswers/texts",
+		csvOutputPath: "./dataset/palianswers/train.csv",
+		includeMetadata: true,
+		metadataFields: ["title", "description", "author"]
+	});
+	if ( enable )
+	{
+		await scraper.start();
+	}
+	return scraper;
+}
 
-async function khameneiIrFreePalestineTag ()
+async function khameneiIrFreePalestineTag ( enable )
 {
 	// https://english.khamenei.ir/Opinions/FreePalestine
 	// https://english.khamenei.ir/page/search.xhtml?topicid=0&period=0&q=FreePalestine&pageSize=100#
@@ -135,6 +163,7 @@ async function electronicintifada ( enable )
 
 async function palestineremembered ( enable )
 {
+	// https://www.palestineremembered.com
 	const scraper = new WebScraper({
 		baseURL: "https://www.palestineremembered.com",
 		startURL: "https://www.palestineremembered.com/ZionistFAQ.html",
@@ -179,24 +208,21 @@ async function palestineremembered ( enable )
 
 void async function main ()
 {
-	const khameneiIrFreePalestineTagScraper = await khameneiIrFreePalestineTag( false );
-	const decolonizepalestineScraper = await decolonizepalestine( false );
+	const palianswersScraper = await palianswers( true );
+	const decolonizepalestineScraper = await decolonizepalestine( true );
+	const khameneiIrFreePalestineTagScraper = await khameneiIrFreePalestineTag( true );
 	const bdsmovementScraper = await bdsmovement( false );
-	const electronicintifadaScraper = await electronicintifada( false );
-	  const palestinerememberedScraper = await palestineremembered( true );
+	const electronicintifadaScraper = await electronicintifada( true );
+	const palestinerememberedScraper = await palestineremembered( false );
+
 	await WebScraper.combineResults( "./dataset/combined", [
-		khameneiIrFreePalestineTagScraper,
+		palianswersScraper,
 		decolonizepalestineScraper,
-		bdsmovementScraper,
+		khameneiIrFreePalestineTagScraper,
 		electronicintifadaScraper,
-		palestinerememberedScraper
+		// bdsmovementScraper,
+		// palestinerememberedScraper,
 	] );
 
-	// 5
-	// https://www.palestineremembered.com/ZionistFAQ.html
-
-	// 6 https://the-palestinian-side.vercel.app/
-
 	// 7 https://stand-with-palestine.org/blogs
 }()
-

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "clean-web-scraper",
-  "version": "3.5.1",
+  "version": "3.5.3",
   "main": "main.js",
   "scripts": {
     "start": "node main.js",

package/src/WebScraper.js

@@ -10,78 +10,79 @@ const { connect } = require( "puppeteer-real-browser" )
 class WebScraper
 {
 	constructor ({
+		// Base configuration
 		baseURL,
 		startURL,
+		strictBaseURL = true,
 		maxDepth = Infinity,
-		maxArticles = Infinity, // Add this line
-		excludeList,
-		exactExcludeList,
+		maxArticles = Infinity,
+
+		// URL filtering
+		excludeList = [],
+		exactExcludeList = [],
+		filterFileTypes = true,
+		excludedFileTypes = [".mp3", ".mp4", ".wav", ".avi", ".mov", ".pdf", ".zip", ".rar"],
+
+		// Output paths
 		scrapResultPath = "./dataset",
 		jsonlOutputPath,
 		textOutputPath,
 		csvOutputPath,
+
+		// Metadata options
 		includeMetadata = false,
-		metadataFields = [], // ['title', 'description', 'author', etc.]
+		metadataFields = [],
+
+		// Network options
 		axiosHeaders,
 		axiosProxy,
+
+		// Puppeteer options
 		usePuppeteer,
 		puppeteerProxy, // e.g. http://127.0.0.1:2080
 		puppeteerExecutablePath,
-		puppeteerRealProxy,
-		filterFileTypes = true,
-		excludedFileTypes = [".mp3", ".mp4", ".wav", ".avi", ".mov", ".pdf", ".zip", ".rar"]
+		puppeteerRealProxy
 	})
 	{
+		// Base configuration
 		this.baseURL = baseURL;
 		this.startURL = startURL || baseURL;
+		this.strictBaseURL = strictBaseURL;
 		this.maxDepth = maxDepth;
-		this.maxArticles = maxArticles; // Add this line
+		this.maxArticles = maxArticles;
+
+		// Output paths setup
 		this.scrapResultPath = scrapResultPath;
 		this.jsonlOutputPath = jsonlOutputPath || path.join( this.scrapResultPath, "train.jsonl" );
 		this.textOutputPath = textOutputPath || path.join( this.scrapResultPath, "texts" );
 		this.csvOutputPath = csvOutputPath || path.join( this.scrapResultPath, "train.csv" );
-		this.jsonlOutputPathWithMeta = jsonlOutputPath.replace( ".jsonl", "_with_metadata.jsonl" );
-		this.csvOutputPathWithMeta = csvOutputPath.replace( ".csv", "_with_metadata.csv" );
-		this.axiosHeaders = axiosHeaders;
+		this.jsonlOutputPathWithMeta = this.jsonlOutputPath.replace( ".jsonl", "_with_metadata.jsonl" );
+		this.csvOutputPathWithMeta = this.csvOutputPath.replace( ".csv", "_with_metadata.csv" );
+
+		// Metadata configuration
 		this.includeMetadata = includeMetadata;
-	   this.metadataFields = new Set( metadataFields );
+		this.metadataFields = new Set( metadataFields );
+
+		// URL filtering setup
 		this.visited = new Set();
 		this.excludeList = this.normalizeExcludeList( excludeList );
 		this.exactExcludeList = this.normalizeExcludeList( exactExcludeList );
-		this.allProcessedContent = [];
 		this.filterFileTypes = filterFileTypes;
 		this.excludedFileTypes = excludedFileTypes;
+
+		// Network configuration
+		this.axiosHeaders = axiosHeaders;
 		this.axiosProxy = axiosProxy;
+
+		// Content storage
+		this.allProcessedContent = [];
+
+		// Puppeteer configuration
 		this.usePuppeteer = usePuppeteer || false;
-		this.puppeteerOptions = {
-			headless: false,
-			userDataDir: "./tmp/browser",
-			defaultViewport: null,
-			args: [
-				"--start-maximized"
-			],
-			"ignoreDefaultArgs": true,
-		}
-		if ( puppeteerProxy )
-		{
-			this.puppeteerOptions.args.push( `--proxy-server=${puppeteerProxy}` );
-		}
-		if ( puppeteerExecutablePath )
-		{
-			this.puppeteerOptions.executablePath = puppeteerExecutablePath;
-		}
-		this.puppeteerRealOptions = {
-			headless: false,
-			args: [],
-			customConfig: {},
-			turnstile: true,
-			connectOption: {},
-			disableXvfb: false,
-			ignoreAllFlags: false,
-			proxy: puppeteerRealProxy
-		}
-		this.puppeteerBrowser = null;
-		this.puppeteerPage = null;
+		this.puppeteerProxy = puppeteerProxy;
+		this.puppeteerExecutablePath = puppeteerExecutablePath;
+		this.puppeteerRealProxy = puppeteerRealProxy;
+		this.configurePuppeteer( );
 	}
 
 	async start ()
@@ -131,6 +132,10 @@ class WebScraper
 		{
 			return;
 		}
+		if ( !this.isValidDomain( url ) )
+		{
+			return;
+		}
 		try
 		{
 			const data = await this.caller( url );
@@ -543,6 +548,41 @@ class WebScraper
 		};
 	}
 
+	configurePuppeteer ( )
+	{
+		this.puppeteerOptions = {
+			headless: false,
+			userDataDir: "./tmp/browser",
+			defaultViewport: null,
+			args: ["--start-maximized"],
+			ignoreDefaultArgs: true
+		};
+
+		if ( this.puppeteerProxy )
+		{
+			this.puppeteerOptions.args.push( `--proxy-server=${this.puppeteerProxy}` );
+		}
+
+		if ( this.puppeteerExecutablePath )
+		{
+			this.puppeteerOptions.executablePath = this.puppeteerExecutablePath;
+		}
+
+		this.puppeteerRealOptions = {
+			headless: false,
+			args: [],
+			customConfig: {},
+			turnstile: true,
+			connectOption: {},
+			disableXvfb: false,
+			ignoreAllFlags: false,
+			proxy: this.puppeteerRealProxy
+		};
+
+		this.puppeteerBrowser = null;
+		this.puppeteerPage = null;
+	}
+
 	normalizeExcludeList ( list = [] )
 	{
 		const normalizedSet = new Set();
@@ -571,36 +611,6 @@ class WebScraper
 		return Array.from( this.excludeList ).some( excluded => { return url.startsWith( excluded ) });
 	}
 
-	createOutputDirectory ()
-	{
-		if ( fs.existsSync( path.join( __dirname, this.scrapResultPath ) ) )
-		{
-			fs.rmSync( path.join( __dirname, this.scrapResultPath ), { recursive: true, force: true });
-		}
-		if ( fs.existsSync( path.join( __dirname, this.textOutputPath ) ) )
-		{
-			fs.rmSync( path.join( __dirname, this.textOutputPath ), { recursive: true, force: true });
-		}
-		if ( fs.existsSync( path.join( __dirname, this.csvOutputPath ) ) )
-		{
-			fs.rmSync( path.join( __dirname, this.csvOutputPath ), { recursive: true, force: true });
-		}
-		if ( fs.existsSync( path.join( __dirname, this.csvOutputPathWithMeta ) ) )
-		{
-			fs.rmSync( path.join( __dirname, this.csvOutputPathWithMeta ), { recursive: true, force: true });
-		}
-		if ( fs.existsSync( path.join( __dirname, this.jsonlOutputPath ) ) )
-		{
-			fs.rmSync( path.join( __dirname, this.jsonlOutputPath ), { recursive: true, force: true });
-		}
-		if ( fs.existsSync( path.join( __dirname, this.jsonlOutputPathWithMeta ) ) )
-		{
-			fs.rmSync( path.join( __dirname, this.jsonlOutputPathWithMeta ), { recursive: true, force: true });
-		}
-		fs.mkdirSync( path.join( __dirname, this.scrapResultPath ), { recursive: true });
-		fs.mkdirSync( path.join( __dirname, this.textOutputPath ), { recursive: true });
-	}
-
 	isValidFileType ( url )
 	{
 		if ( !this.filterFileTypes ) return true;
@@ -608,6 +618,22 @@ class WebScraper
 		return !this.excludedFileTypes.some( ext => { return urlPath.endsWith( ext ) });
 	}
 
+	isValidDomain ( url )
+	{
+		if ( !this.strictBaseURL ) return true;
+		try
+		{
+			const urlObj = new URL( url );
+			const baseURLObj = new URL( this.baseURL );
+			return urlObj.hostname === baseURLObj.hostname;
+		}
+		catch ( e )
+		{
+			console.log( `Invalid URL: ${url}` );
+			return false;
+		}
+	}
+
 	isValidContent ( content )
 	{
 		// Remove whitespace and newlines for checking
@@ -635,6 +661,36 @@ class WebScraper
 		return true;
 	}
 
+	createOutputDirectory ()
+	{
+		if ( fs.existsSync( path.join( __dirname, this.scrapResultPath ) ) )
+		{
+			fs.rmSync( path.join( __dirname, this.scrapResultPath ), { recursive: true, force: true });
+		}
+		if ( fs.existsSync( path.join( __dirname, this.textOutputPath ) ) )
+		{
+			fs.rmSync( path.join( __dirname, this.textOutputPath ), { recursive: true, force: true });
+		}
+		if ( fs.existsSync( path.join( __dirname, this.csvOutputPath ) ) )
+		{
+			fs.rmSync( path.join( __dirname, this.csvOutputPath ), { recursive: true, force: true });
+		}
+		if ( fs.existsSync( path.join( __dirname, this.csvOutputPathWithMeta ) ) )
+		{
+			fs.rmSync( path.join( __dirname, this.csvOutputPathWithMeta ), { recursive: true, force: true });
+		}
+		if ( fs.existsSync( path.join( __dirname, this.jsonlOutputPath ) ) )
+		{
+			fs.rmSync( path.join( __dirname, this.jsonlOutputPath ), { recursive: true, force: true });
+		}
+		if ( fs.existsSync( path.join( __dirname, this.jsonlOutputPathWithMeta ) ) )
+		{
+			fs.rmSync( path.join( __dirname, this.jsonlOutputPathWithMeta ), { recursive: true, force: true });
+		}
+		fs.mkdirSync( path.join( __dirname, this.scrapResultPath ), { recursive: true });
+		fs.mkdirSync( path.join( __dirname, this.textOutputPath ), { recursive: true });
+	}
+
 	static sleep ( ms )
 	{
 		return new Promise( resolve => { return setTimeout( resolve, ms ) });