clean-web-scraper 3.1.0 → 3.2.1

package/README.md

@@ -57,10 +57,10 @@ const scraper = new WebScraper({
   includeMetadata: false,                       // Optional: Include metadata in output files
   metadataFields: ['title', 'description']      // Optional: Specify metadata fields to include
 });
-scraper.start();
+await scraper.start();
 
 // Combine results from multiple scrapers
-WebScraper.combineResults('./combined-dataset', [scraper1, scraper2]);
+await WebScraper.combineResults('./combined-dataset', [scraper1, scraper2]);
 ```
 
 ```bash

package/example-usage.js

@@ -58,7 +58,7 @@ void async function main ()
 {
 	const khameneiIrFreePalestineTagScraper = await khameneiIrFreePalestineTag();
 	const decolonizepalestineScraper = await decolonizepalestine();
-	WebScraper.combineResults( "./dataset/combined", [
+	await WebScraper.combineResults( "./dataset/combined", [
 		khameneiIrFreePalestineTagScraper,
 		decolonizepalestineScraper
 	] );

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "clean-web-scraper",
-  "version": "3.1.0",
+  "version": "3.2.1",
   "main": "main.js",
   "scripts": {
     "start": "node main.js",

package/src/WebScraper.js

@@ -155,9 +155,14 @@ class WebScraper
 	{
 		const writeStreamSimple = fs.createWriteStream( path.join( __dirname, this.jsonlOutputPath ) );
 		let writeStreamMeta
+
+		// Add error handlers
+		writeStreamSimple.on( "error", ( err ) => { return console.error( "Error writing JSONL:", err ) });
+
 		if ( this.includeMetadata )
 		{
 			writeStreamMeta = fs.createWriteStream( path.join( __dirname, this.jsonlOutputPathWithMeta ) );
+			writeStreamMeta.on( "error", ( err ) => { return console.error( "Error writing metadata JSONL:", err ) });
 		}
 		for ( const content of this.allProcessedContent )
 		{
@@ -171,6 +176,7 @@ class WebScraper
 		if ( this.includeMetadata )
 		{
 			writeStreamMeta.end();
+			console.log( `Created JSONL file at: ${this.jsonlOutputPathWithMeta}` );
 		}
 		console.log( `Created JSONL file at: ${this.jsonlOutputPath}` );
 	}
@@ -179,6 +185,7 @@ class WebScraper
 	{
 		// Create simple version
 		const writeStreamSimple = fs.createWriteStream( path.join( __dirname, this.csvOutputPath ) );
+		writeStreamSimple.on( "error", ( err ) => { return console.error( "Error writing CSV:", err ) });
 		writeStreamSimple.write( "text\n" );
 
 		// Create metadata version if requested
@@ -186,6 +193,7 @@ class WebScraper
 		if ( this.includeMetadata )
 		{
 			writeStreamMeta = fs.createWriteStream( path.join( __dirname, this.csvOutputPathWithMeta ) );
+			writeStreamMeta.on( "error", ( err ) => { return console.error( "Error writing metadata CSV:", err ) });
 		}
 
 		if ( this.includeMetadata )
@@ -365,8 +373,14 @@ class WebScraper
 		fs.mkdirSync( path.join( __dirname, this.textOutputPath ), { recursive: true });
 	}
 
-	static combineResults ( outputPath, websites )
+	static sleep ( ms )
+	{
+		return new Promise( resolve => { return setTimeout( resolve, ms ) });
+	}
+
+	static async combineResults ( outputPath, websites )
 	{
+		await WebScraper.sleep( 1000 );
 		const fullOutputPath = path.join( __dirname, outputPath );
 
 		// Create output directories
@@ -375,8 +389,10 @@ class WebScraper
 		fs.mkdirSync( path.join( fullOutputPath, "texts_with_metadata" ), { recursive: true });
 
 		// Combine regular JSONL files
-		const jsonlOutput = fs.createWriteStream( path.join( fullOutputPath, "combined.jsonl" ) );
-		const jsonlMetaOutput = fs.createWriteStream( path.join( fullOutputPath, "combined_with_metadata.jsonl" ) );
+		const jsonlOutput = fs.createWriteStream( path.join( fullOutputPath, "combined.jsonl" ) )
+		    .on( "error", ( err ) => { return console.error( "Error combining JSONL:", err ) });
+		const jsonlMetaOutput = fs.createWriteStream( path.join( fullOutputPath, "combined_with_metadata.jsonl" ) )
+		    .on( "error", ( err ) => { return console.error( "Error combining metadata JSONL:", err ) });
 		const csvOutput = fs.createWriteStream( path.join( fullOutputPath, "combined.csv" ) );
 		const csvMetaOutput = fs.createWriteStream( path.join( fullOutputPath, "combined_with_metadata.csv" ) );
 
@@ -449,7 +465,6 @@ class WebScraper
 			}
 		}
 	}
-
 }
 
 module.exports = WebScraper;