clean-web-scraper 3.8.7 → 3.9.1

package/example-usage.js

@@ -129,9 +129,9 @@ async function electronicintifada ( enable )
 		csvOutputPath: "./dataset/electronicintifada/train.csv",
 		includeMetadata: true,
 		metadataFields: ["author", "title", "description", "dateScrapedDate"],
-		maxDepth: 13,
+		maxDepth: 16,
 		maxArticles: 2000,
-		concurrencyLimit: 3,
+		concurrencyLimit: 2,
 		axiosHeaders: headers,
 		retryDelay: 10000,
 		axiosProxy: {
@@ -198,18 +198,19 @@ async function mondoweiss ( enable )
 		textOutputPath: "./dataset/mondoweiss/texts",
 		csvOutputPath: "./dataset/mondoweiss/train.csv",
 		maxArticles: 2500,
-		maxRetries: 2,
-		concurrencyLimit: 4,
+		maxRetries: 3,
+		concurrencyLimit: 3,
 		axiosHeaders: headers,
 		axiosProxy: {
 			host: "localhost",
 			port: 2080,
 			protocol: "http"
 		},
-		maxDepth: 10,
+		maxDepth: 15,
 		retryDelay: 10000,
 		includeMetadata: true,
-		metadataFields: ["author", "title", "description", "dateScrapedDate"]
+		metadataFields: ["author", "title", "description", "dateScrapedDate"],
+		useProxyAsFallback: true
 	});
 	if ( enable )
 	{

package/main.js

@@ -3,49 +3,11 @@ const { JSDOM } = require( "jsdom" );
 const { Readability } = require( "@mozilla/readability" );
 const fs = require( "fs" );
 const path = require( "path" );
-const { connect } = require( "puppeteer-real-browser" )
+const { connect } = require( "puppeteer-real-browser" );
 
 class WebScraper
 {
-	constructor ({
-		// Base configuration
-		baseURL,
-		startURL,
-		strictBaseURL,
-		maxDepth,
-		maxArticles,
-		concurrencyLimit,
-		maxRetries,
-		retryDelay,
-
-		// URL filtering
-		excludeList = [],
-		exactExcludeList = [],
-		filterFileTypes,
-		excludedFileTypes,
-		removeURLFragment,
-
-		// Output paths
-		scrapResultPath = "./dataset",
-		jsonlOutputPath,
-		textOutputPath,
-		csvOutputPath,
-
-		// Metadata options
-		includeMetadata = false,
-		metadataFields = [],
-
-		// Network options
-		axiosHeaders,
-		axiosProxy,
-		useProxyAsFallback,
-
-		// Puppeteer options
-		usePuppeteer,
-		puppeteerProxy, // e.g. http://127.0.0.1:2080
-		puppeteerExecutablePath,
-		puppeteerRealProxy
-	})
+	constructor ( config = {})
 	{
 		// Base configuration
 		this.baseURL = baseURL;
@@ -54,11 +16,10 @@ class WebScraper
 		this.maxDepth = maxDepth || Infinity;
 		this.maxArticles = maxArticles || Infinity;
 		this.concurrencyLimit = concurrencyLimit || 2;
-		this.maxRetries = maxRetries || 5;
-		this.retryDelay = retryDelay || 40000;
+		this.crawlingDelay = crawlingDelay || 1000;
 
 		// Output paths setup
-		this.scrapResultPath = scrapResultPath;
+		this.scrapResultPath = scrapResultPath || "./dataset";
 		this.textOutputPath = textOutputPath || path.join( this.scrapResultPath, "texts" );
 		this.textOutputPathWithMeta = `${this.textOutputPath }_with_metadata`;
 		this.jsonlOutputPath = jsonlOutputPath || path.join( this.scrapResultPath, "train.jsonl" );
@@ -67,8 +28,8 @@ class WebScraper
 		this.csvOutputPathWithMeta = this.csvOutputPath.replace( ".csv", "_with_metadata.csv" );
 
 		// Metadata configuration
-		this.includeMetadata = includeMetadata;
-		this.metadataFields = new Set( metadataFields );
+		this.includeMetadata = includeMetadata || false;
+		this.metadataFields = new Set( metadataFields || [] );
 
 		// URL filtering setup
 		this.visited = new Set();
@@ -81,6 +42,8 @@ class WebScraper
 		// Network configuration
 		this.axiosHeaders = axiosHeaders;
 		this.axiosProxy = axiosProxy;
+		this.axiosMaxRetries = axiosMaxRetries || 5;
+		this.axiosRetryDelay = axiosRetryDelay || 40000;
 		this.useProxyAsFallback = useProxyAsFallback || false;
 		this.axiosOptions = {};
 		if ( this.axiosHeaders )
@@ -97,10 +60,10 @@ class WebScraper
 
 		// Puppeteer configuration
 		this.usePuppeteer = usePuppeteer || false;
-		this.puppeteerProxy = puppeteerProxy;
+		this.puppeteerProxy = puppeteerProxy; // http://127.0.0.1:2080
 		this.puppeteerExecutablePath = puppeteerExecutablePath;
 		this.puppeteerRealProxy = puppeteerRealProxy;
-		this.configurePuppeteer( );
+		this.configurePuppeteer();
 	}
 
 	async start ()
@@ -109,12 +72,12 @@ class WebScraper
 		{
 			if ( this.usePuppeteer )
 			{
-				let { browser, page } = await connect( this.puppeteerRealOptions )
+				const { browser, page } = await connect( this.puppeteerRealOptions );
 				this.puppeteerBrowser = browser;
 				this.puppeteerPage = page;
 			}
 			this.createOutputDirectory();
-			await this.fetchPage( this.startURL, 0 );
+			await this.crawl( this.startURL, 0 );
 			this.createJSONLFile();
 			this.saveNumberedTextFiles();
 			this.createCSVFile();
@@ -129,94 +92,95 @@ class WebScraper
 		{
 			if ( this.puppeteerBrowser )
 			{
-				await this.puppeteerBrowser.close(); // Close the browser after scraping
+				await this.puppeteerBrowser.close();
 			}
 		}
 	}
 
-	async fetchPage ( url, depth )
+	async crawl ( initialUrl, initialDepth = 0 )
 	{
-		if ( this.removeURLFragment )
-		{
-			url = url.split( "#" )[0];
-		}
-		if ( this.hasReachedMax( depth ) )
-		{
-			return;
-		}
-		if ( this.visited.has( url ) )
-		{
-			console.log( `Already visited: ${url}` );
-			return;
-		}
-		this.visited.add( url );
-		if ( !this.isValidFileType( url ) || !this.isValidDomain( url ) )
-		{
-			return;
-		}
-		try
+		const queue = [{ url: initialUrl, depth: initialDepth }];
+		for ( let i = 0; i < queue.length; i++ )
 		{
-			await WebScraper.sleep( 5000 );
-			const data = await this.fetchContent( url );
-			if ( !data ) return;
-			const dom = new JSDOM( data, { url });
-			const { document } = dom.window;
+			let { url, depth } = queue[i];
+			console.log( `Processing URL: ${queue[i].url}` );
+			if ( this.hasReachedMax( depth ) )
+			{
+				continue;
+			}
+			if ( this.removeURLFragment )
+			{
+				url = url.split( "#" )[0];
+			}
+			if ( this.visited.has( url ) )
+			{
+				console.log( `Already visited: ${url}` );
+				continue;
+			}
+			this.visited.add( url );
 
-			if ( !this.isExcluded( url ) )
+			if ( !this.isValidFileType( url ) || !this.isValidDomain( url ) )
 			{
-				const reader = new Readability( document, { charThreshold: 500, nbTopCandidates: 20 });
-				const article = reader.parse();
+				continue;
+			}
 
-				if ( article )
+			try
+			{
+				await WebScraper.sleep( this.crawlingDelay );
+				const data = await this.fetchContent( url );
+				if ( !data ) continue;
+
+				const dom = new JSDOM( data, { url });
+				const { document } = dom.window;
+
+				if ( !this.isExcluded( url ) )
 				{
-					if ( this.hasValidPageContent( article.textContent ) )
+					const reader = new Readability( document, {
+						charThreshold: 500,
+						nbTopCandidates: 20
+					});
+					const article = reader.parse();
+					if ( article )
 					{
-						const metadata = this.extractMetadata( url, document );
-						this.saveArticle( url, article.textContent, metadata );
+						if ( this.hasValidPageContent( article.textContent ) )
+						{
+							const metadata = this.extractMetadata( url, document );
+							this.saveArticle( url, article.textContent, metadata );
+						}
+						else
+						{
+							console.error( `Invalid content found at ${url}` );
+						}
 					}
 					else
 					{
-						console.error( `Invalid content found at ${url}` );
+						console.error( `No readable content found at ${url}` );
 					}
 				}
-				else
-				{
-					console.error( `No readable content found at ${url}` );
-				}
-			}
-
-			const links = this.extractLinks( data );
-			const unvisitedLinks = Array.from( links ).filter( link => { return !this.visited.has( link ) });
-
-			for ( let i = 0; i < unvisitedLinks.length; i += this.concurrencyLimit )
-			{
-				if ( this.hasReachedMax( depth ) )
-				{
-					return;
-				}
-				const batch = unvisitedLinks.slice( i, i + this.concurrencyLimit );
-				const results = await Promise.allSettled( batch.map( link => { return this.fetchPage( link, depth + 1 ) }) );
 
-				results.forEach( ( result, index ) =>
+				const links = this.extractLinks( data );
+				const unvisitedLinks = Array.from( links ).filter( link => { return !this.visited.has( link ) });
+				for ( const link of unvisitedLinks )
 				{
-					if ( result.status === "rejected" )
+					if ( !this.hasReachedMax( depth ) )
 					{
-						console.error( `Failed to fetch ${batch[index]}: ${result.reason}` );
+						queue.push({ url: link, depth: depth + 1 });
 					}
-				});
+				}
+			}
+			catch ( error )
+			{
+				console.error( `Error fetching ${url}:`, error.message, error.code );
 			}
-		}
-		catch ( error )
-		{
-			console.error( `Error fetching ${url}:`, error.message, error.code );
 		}
 	}
 
+
 	async fetchContent ( url )
 	{
 		try
 		{
-			const response = await this.retryAxiosRequest( url )
+			const response = await this.retryAxiosRequest( url );
 			const contentType = response?.headers["content-type"] || "";
 			if ( !contentType?.startsWith( "text" ) )
 			{
@@ -225,34 +189,30 @@ class WebScraper
 				return null;
 			}
 
-			// Step 3: If Content-Type is HTML, read the full response
 			let htmlContent = "";
-			response.data.on( "data", ( chunk ) =>
+			response.data.on( "data", chunk =>
 			{
 				htmlContent += chunk.toString();
 			});
-
-			// Wait for the stream to finish
 			await new Promise( ( resolve, reject ) =>
 			{
 				response.data.on( "end", resolve );
 				response.data.on( "error", reject );
 			});
-
 			return htmlContent;
 		}
 		catch ( error )
 		{
 			console.error( `Error fetching content ${url}:`, error.message );
-			if ( error.status = 403 && this.usePuppeteer )
+			if ( error.status === 403 && this.usePuppeteer )
 			{
 				try
 				{
 					let result;
-					for ( let index = 0; index < 10; index++ )
+					for ( let i = 0; i < 10; i++ )
 					{
 						console.log( `Please solve the CAPTCHA on the opened browser window for ${url}` );
-						result = await this.navigateToPage( url ) ;
+						result = await this.navigateToPage( url );
 						if ( this.hasValidPageContent( result.htmlContent ) )
 						{
 							break
@@ -265,7 +225,6 @@ class WebScraper
 					console.error( `Error solving CAPTCHA for ${url}:`, error.message, error );
 					throw error;
 				}
-
 			}
 			throw error;
 		}
@@ -378,11 +337,10 @@ class WebScraper
 	createJSONLFile ()
 	{
 		const writeStreamSimple = fs.createWriteStream( path.join( __dirname, this.jsonlOutputPath ) );
-		let writeStreamMeta
-
-		// Add error handlers
-		writeStreamSimple.on( "error", ( err ) => { return console.error( "Error writing JSONL:", err ) });
+		writeStreamSimple.on( "error", err =>
+		{ return console.error( "Error writing JSONL:", err ) });
 
+		let writeStreamMeta;
 		if ( this.includeMetadata )
 		{
 			writeStreamMeta = fs.createWriteStream( path.join( __dirname, this.jsonlOutputPathWithMeta ) );
@@ -428,16 +386,18 @@ class WebScraper
 
 		for ( const content of this.allProcessedContent )
 		{
-			// Write simple version
 			const escapedText = content.simple.text.replace( /"/g, "\"\"" );
 			writeStreamSimple.write( `"${escapedText}"\n` );
 
-			// Write metadata version if requested
 			if ( this.includeMetadata )
 			{
 				const { metadata } = content.withMetadata;
-				const metadataValues = Array.from( this.metadataFields )
-				.map( field => { return metadata[field] ? `"${metadata[field].replace( /"/g, "\"\"" )}"` : "\"\"" });
+				const metadataValues = Array.from( this.metadataFields ).map( field =>
+				{
+					return metadata[field]
+						? `"${metadata[field].replace( /"/g, "\"\"" )}"`
+						: "\"\""
+				});
 				writeStreamMeta.write( `"${escapedText}",${metadataValues.join( "," )}\n` );
 			}
 		}
@@ -456,10 +416,8 @@ class WebScraper
 
 	saveNumberedTextFiles ()
 	{
-		// Create base text folder for simple content
 		const baseTextPath = path.join( __dirname, this.textOutputPath );
 
-		// Create metadata text folder if needed
 		let metaTextPath = null;
 		if ( this.includeMetadata )
 		{
@@ -470,20 +428,15 @@ class WebScraper
 		this.allProcessedContent.forEach( ( content, index ) =>
 		{
 			const fileName = `${index + 1}.txt`;
-
-			// Always save simple version
 			const simpleFilePath = path.join( baseTextPath, fileName );
 			fs.writeFileSync( simpleFilePath, content.simple.text, "utf-8" );
 			console.log( `Created simple text file: ${fileName}` );
 
-			// Save metadata version if enabled
 			if ( this.includeMetadata )
 			{
 				const metaFilePath = path.join( metaTextPath, fileName );
 				let fileContent = "";
-
 				const { metadata } = content.withMetadata;
-				// Add metadata fields as headers
 				for ( const field of this.metadataFields )
 				{
 					if ( metadata[field] )
@@ -493,7 +446,6 @@ class WebScraper
 				}
 				fileContent += "\n---\n\n";
 				fileContent += content.withMetadata.text;
-
 				fs.writeFileSync( metaFilePath, fileContent, "utf-8" );
 				console.log( `Created metadata text file: ${fileName}` );
 			}
@@ -503,33 +455,27 @@ class WebScraper
 	processContent ( content )
 	{
 		let processed = content;
-
-		// Remove "[You can read more about this here]" and similar patterns
+		// Remove unwanted fixed text
 		processed = processed.replace( /\[You can read more about this here\]/g, "" ).trim();
-
-		// Trim each line
-		processed = processed.split( "\n" )
+		// Trim each line and remove extra newlines
+		processed = processed
+		.split( "\n" )
 		.map( line => { return line.trim() })
-		.join( "\n" );
-
-		// Replace 3 or more newlines with a single newline
-		processed = processed.replace( /\n{3,}/g, "\n\n" );
+		.join( "\n" )
+		.replace( /\n{3,}/g, "\n\n" );
 
-		// Add more processing rules as needed:
-		// processed = processed.replace(/\[.*?\]/g, ''); // Removes all content within square brackets
-		// processed = processed.replace(/\(.*?\)/g, ''); // Removes all content within parentheses
-
-		// Remove specified words from the end of content, handling multiple occurrences
+		// Remove specified words at the end (repeatedly)
 		const wordsToTrim = ["Facebook", "Twitter", "Donate Now", "Instagram"];
 		let changed = true;
-
 		while ( changed )
 		{
 			changed = false;
-			for ( let i = 0; i < wordsToTrim.length; i++ )
+			for ( const word of wordsToTrim )
 			{
 				const oldProcessed = processed;
-				processed = processed.replace( new RegExp( `\\s*${wordsToTrim[i]}\\s*$`, "g" ), "" ).trim();
+				processed = processed
+				.replace( new RegExp( `\\s*${word}\\s*$`, "g" ), "" )
+				.trim();
 				if ( oldProcessed !== processed )
 				{
 					changed = true;
@@ -542,7 +488,6 @@ class WebScraper
 	filterMetadata ( metadata )
 	{
 		if ( !this.includeMetadata ) return {};
-
 		const filteredMetadata = {};
 		for ( const field of this.metadataFields )
 		{
@@ -562,10 +507,13 @@ class WebScraper
 			description: document.querySelector( "meta[name=\"description\"]" )?.content,
 			keywords: document.querySelector( "meta[name=\"keywords\"]" )?.content,
 			author: document.querySelector( "meta[name=\"author\"]" )?.content,
-			language: document.documentElement.lang || document.querySelector( "html" )?.getAttribute( "lang" ),
+			language:
+        document.documentElement.lang ||
+        document.querySelector( "html" )?.getAttribute( "lang" ),
 			canonicalUrl: document.querySelector( "link[rel=\"canonical\"]" )?.href,
 			ogTitle: document.querySelector( "meta[property=\"og:title\"]" )?.content,
-			ogDescription: document.querySelector( "meta[property=\"og:description\"]" )?.content,
+			ogDescription: document.querySelector( "meta[property=\"og:description\"]" )
+			?.content,
 			ogImage: document.querySelector( "meta[property=\"og:image\"]" )?.content,
 			ogType: document.querySelector( "meta[property=\"og:type\"]" )?.content,
 			dateScrapedDate: new Date().toISOString()
@@ -581,7 +529,7 @@ class WebScraper
 			...this.axiosOptions,
 		};
 
-		for ( let attempt = 1; attempt <= this.maxRetries; attempt++ )
+		for ( let attempt = 1; attempt <= this.axiosMaxRetries; attempt++ )
 		{
 			try
 			{
@@ -589,7 +537,7 @@ class WebScraper
 				{
 					break;
 				}
-				if ( attempt === this.maxRetries && this.useProxyAsFallback && this.axiosProxy )
+				if ( attempt === this.axiosMaxRetries && this.useProxyAsFallback && this.axiosProxy )
 				{
 					options = {
 						...options,
@@ -601,9 +549,9 @@ class WebScraper
 			}
 			catch ( error )
 			{
-				if ( attempt >= this.maxRetries ) throw error;
-				await WebScraper.sleep( this.retryDelay * attempt );
-				console.error( `Retrying request to ${url} (Attempt ${attempt + 1}/${this.maxRetries})`, error.message, error.code );
+				if ( attempt >= this.axiosMaxRetries ) throw error;
+				await WebScraper.sleep( this.axiosRetryDelay * attempt );
+				console.error( `Retrying request to ${url} (Attempt ${attempt + 1}/${this.axiosMaxRetries})`, error.message, error.code );
 			}
 		}
 		throw new Error( "Max retries reached" );
@@ -623,7 +571,6 @@ class WebScraper
 		{
 			this.puppeteerOptions.args.push( `--proxy-server=${this.puppeteerProxy}` );
 		}
-
 		if ( this.puppeteerExecutablePath )
 		{
 			this.puppeteerOptions.executablePath = this.puppeteerExecutablePath;
@@ -697,10 +644,7 @@ class WebScraper
 
 	hasValidPageContent ( content )
 	{
-		// Remove whitespace and newlines for checking
 		const cleanContent = content.replace( /\s+/g, " " ).trim().toLowerCase();
-
-		// List of phrases that indicate invalid content
 		const invalidPhrases = [
 			"verifying that you are not a robot",
 			"verifying you are human. this may take a few seconds.",
@@ -724,37 +668,35 @@ class WebScraper
 
 	createOutputDirectory ()
 	{
-		if ( fs.existsSync( path.join( __dirname, this.scrapResultPath ) ) )
-		{
-			fs.rmSync( path.join( __dirname, this.scrapResultPath ), { recursive: true, force: true });
-		}
-		if ( fs.existsSync( path.join( __dirname, this.textOutputPath ) ) )
-		{
-			fs.rmSync( path.join( __dirname, this.textOutputPath ), { recursive: true, force: true });
-		}
-		if ( fs.existsSync( path.join( __dirname, this.textOutputPathWithMeta ) ) )
-		{
-			fs.rmSync( path.join( __dirname, this.textOutputPathWithMeta ), { recursive: true, force: true });
-		}
-		if ( fs.existsSync( path.join( __dirname, this.csvOutputPath ) ) )
-		{
-			fs.rmSync( path.join( __dirname, this.csvOutputPath ), { recursive: true, force: true });
-		}
-		if ( fs.existsSync( path.join( __dirname, this.csvOutputPathWithMeta ) ) )
-		{
-			fs.rmSync( path.join( __dirname, this.csvOutputPathWithMeta ), { recursive: true, force: true });
-		}
-		if ( fs.existsSync( path.join( __dirname, this.jsonlOutputPath ) ) )
+		const paths = [
+			path.join( __dirname, this.scrapResultPath ),
+			path.join( __dirname, this.textOutputPath ),
+			path.join( __dirname, this.textOutputPathWithMeta ),
+			path.join( __dirname, this.csvOutputPath ),
+			path.join( __dirname, this.csvOutputPathWithMeta ),
+			path.join( __dirname, this.jsonlOutputPath ),
+			path.join( __dirname, this.jsonlOutputPathWithMeta )
+		];
+		for ( const p of paths )
 		{
-			fs.rmSync( path.join( __dirname, this.jsonlOutputPath ), { recursive: true, force: true });
+			if ( fs.existsSync( p ) )
+			{
+				fs.rmSync( p, { recursive: true, force: true });
+			}
 		}
-		if ( fs.existsSync( path.join( __dirname, this.jsonlOutputPathWithMeta ) ) )
+		// Recreate directories needed for output
+		this.ensureDirectory( path.join( __dirname, this.scrapResultPath ) );
+		this.ensureDirectory( path.join( __dirname, this.textOutputPath ) );
+		this.ensureDirectory( path.join( __dirname, this.textOutputPathWithMeta ) );
+	}
+
+	// Helper method to ensure a directory exists
+	ensureDirectory ( dirPath )
+	{
+		if ( !fs.existsSync( dirPath ) )
 		{
-			fs.rmSync( path.join( __dirname, this.jsonlOutputPathWithMeta ), { recursive: true, force: true });
+			fs.mkdirSync( dirPath, { recursive: true });
 		}
-		fs.mkdirSync( path.join( __dirname, this.scrapResultPath ), { recursive: true });
-		fs.mkdirSync( path.join( __dirname, this.textOutputPath ), { recursive: true });
-		fs.mkdirSync( path.join( __dirname, this.textOutputPathWithMeta ), { recursive: true });
 	}
 
 	static async sleep ( ms )
@@ -766,11 +708,7 @@ class WebScraper
 	{
 		await WebScraper.sleep( 1000 );
 		const fullOutputPath = path.join( __dirname, outputPath );
-
-		// Create output directories
 		WebScraper.createCombinedDirectories( fullOutputPath );
-
-		// Combine files by type
 		WebScraper.combineJSONLFiles( fullOutputPath, websites );
 		WebScraper.combineCSVFiles( fullOutputPath, websites );
 		WebScraper.combineTextFiles( fullOutputPath, websites );
@@ -784,34 +722,44 @@ class WebScraper
 		}
 		fs.mkdirSync( fullOutputPath, { recursive: true });
 		fs.mkdirSync( path.join( fullOutputPath, "texts" ), { recursive: true });
-		fs.mkdirSync( path.join( fullOutputPath, "texts_with_metadata" ), { recursive: true });
+		fs.mkdirSync( path.join( fullOutputPath, "texts_with_metadata" ), {
+			recursive: true
+		});
 	}
 
 	static combineJSONLFiles ( fullOutputPath, websites )
 	{
-		const jsonlOutput = fs.createWriteStream( path.join( fullOutputPath, "combined.jsonl" ) )
-		.on( "error", ( err ) => { return console.error( "Error combining JSONL:", err ) });
-		const jsonlMetaOutput = fs.createWriteStream( path.join( fullOutputPath, "combined_with_metadata.jsonl" ) )
-		.on( "error", ( err ) => { return console.error( "Error combining metadata JSONL:", err ) });
+		const jsonlOutput = fs
+		.createWriteStream( path.join( fullOutputPath, "combined.jsonl" ) )
+		.on( "error", err =>
+		{ return console.error( "Error combining JSONL:", err ) });
+		const jsonlMetaOutput = fs
+		.createWriteStream( path.join( fullOutputPath, "combined_with_metadata.jsonl" ) )
+		.on( "error", err =>
+		{ return console.error( "Error combining metadata JSONL:", err ) });
 
 		for ( const website of websites )
 		{
-			const jsonlContent = fs.readFileSync( path.join( __dirname, website.jsonlOutputPath ), "utf-8" );
+			const jsonlContent = fs.readFileSync(
+				path.join( __dirname, website.jsonlOutputPath ),
+				"utf-8"
+			);
 			if ( jsonlContent )
 			{
 				jsonlOutput.write( jsonlContent );
 			}
-
 			if ( website.includeMetadata )
 			{
-				const jsonlMetaContent = fs.readFileSync( path.join( __dirname, website.jsonlOutputPathWithMeta ), "utf-8" );
+				const jsonlMetaContent = fs.readFileSync(
+					path.join( __dirname, website.jsonlOutputPathWithMeta ),
+					"utf-8"
+				);
 				if ( jsonlMetaContent )
 				{
 					jsonlMetaOutput.write( jsonlMetaContent );
 				}
 			}
 		}
-
 		jsonlOutput.end();
 		jsonlMetaOutput.end();
 	}
@@ -822,7 +770,8 @@ class WebScraper
 		const csvMetaOutput = fs.createWriteStream( path.join( fullOutputPath, "combined_with_metadata.csv" ) );
 
 		csvOutput.write( "text\n" );
-		const metadataFields = websites.find( w => { return w.includeMetadata })?.metadataFields || new Set();
+		const metadataFields =
+      websites.find( w => { return w.includeMetadata })?.metadataFields || new Set();
 
 		if ( metadataFields.size > 0 )
 		{
@@ -839,10 +788,13 @@ class WebScraper
 			{
 				csvOutput.write( `${csvContent.join( "\n" )}\n` );
 			}
-
 			if ( website.includeMetadata )
 			{
-				const csvMetaContent = fs.readFileSync( path.join( __dirname, website.csvOutputPathWithMeta ), "utf-8" )
+				const csvMetaContent = fs
+				.readFileSync(
+					path.join( __dirname, website.csvOutputPathWithMeta ),
+					"utf-8"
+				)
 				.split( "\n" )
 				.slice( 1 )
 				.filter( line => { return line.trim() });
@@ -852,7 +804,6 @@ class WebScraper
 				}
 			}
 		}
-
 		csvOutput.end();
 		csvMetaOutput.end();
 	}
@@ -860,20 +811,20 @@ class WebScraper
 	static combineTextFiles ( fullOutputPath, websites )
 	{
 		let textFileCounter = 1;
-
 		for ( const website of websites )
 		{
 			const textFiles = fs.readdirSync( path.join( __dirname, website.textOutputPath ) );
-
 			for ( const file of textFiles )
 			{
-				const content = fs.readFileSync( path.join( __dirname, website.textOutputPath, file ), "utf-8" );
+				const content = fs.readFileSync(
+					path.join( __dirname, website.textOutputPath, file ),
+					"utf-8"
+				);
 				fs.writeFileSync(
 					path.join( fullOutputPath, "texts", `${textFileCounter}.txt` ),
 					content,
 					"utf-8"
 				);
-
 				if ( website.includeMetadata )
 				{
 					const metaContent = fs.readFileSync(
@@ -881,7 +832,11 @@ class WebScraper
 						"utf-8"
 					);
 					fs.writeFileSync(
-						path.join( fullOutputPath, "texts_with_metadata", `${textFileCounter}.txt` ),
+						path.join(
+							fullOutputPath,
+							"texts_with_metadata",
+							`${textFileCounter}.txt`
+						),
 						metaContent,
 						"utf-8"
 					);
@@ -892,4 +847,4 @@ class WebScraper
 	}
 }
 
-module.exports = WebScraper;
+module.exports = WebScraper;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "clean-web-scraper",
-  "version": "3.8.7",
+  "version": "3.9.1",
   "main": "main.js",
   "scripts": {
     "start": "node main.js",