clean-web-scraper 3.2.2 → 3.2.4

package/README.md

@@ -53,9 +53,6 @@ const scraper = new WebScraper({
   jsonlOutputPath: './example.com/train.jsonl', // Optional: Custom JSONL output path
   textOutputPath: "./example.com/texts",        // Optional: Custom text output path
   csvOutputPath: "./example.com/train.csv",     // Optional: Custom CSV output path
-  maxDepth: 3,                                  // Optional: Maximum depth for recursive crawling
-  includeMetadata: false,                       // Optional: Include metadata in output files
-  metadataFields: ['title', 'description']      // Optional: Specify metadata fields to include
 });
 await scraper.start();
 ```
@@ -72,13 +69,19 @@ const WebScraper = require('clean-web-scraper');
 // Scrape documentation website
 const docsScraper = new WebScraper({
   baseURL: 'https://docs.example.com',
-  scrapResultPath: './datasets/docs'
+  scrapResultPath: './datasets/docs',
+  maxDepth: 3,                               // Optional: Maximum depth for recursive crawling
+  includeMetadata: true,                     // Optional: Include metadata in output files
+  metadataFields: ['title', 'description']   // Optional: Specify metadata fields to include
 });
 
 // Scrape blog website
 const blogScraper = new WebScraper({
   baseURL: 'https://blog.example.com',
-  scrapResultPath: './datasets/blog'
+  scrapResultPath: './datasets/blog',
+  maxDepth: 3,                               // Optional: Maximum depth for recursive crawling
+  includeMetadata: true,                     // Optional: Include metadata in output files
+  metadataFields: ['title', 'description']   // Optional: Specify metadata fields to include
 });
 
 // Start scraping both sites
@@ -86,7 +89,7 @@ await docsScraper.start();
 await blogScraper.start();
 
 // Combine all scraped content into a single dataset
-await WebScraper.combineResults('./combined-dataset', [docsScraper, blogScraper]);
+await WebScraper.combineResults('./combined', [docsScraper, blogScraper]);
 ```
 
 ## 📤 Output
@@ -118,6 +121,79 @@ example.com/
 ├── train_with_metadata.jsonl  # When includeMetadata is true
 ├── train.csv             # Clean text in CSV format
 └── train_with_metadata.csv    # When includeMetadata is true
+
+combined/
+├── texts/                # Combined numbered text files
+│   ├── 1.txt
+│   ├── 2.txt
+│   └── n.txt
+├── texts_with_metadata/  # Combined metadata text files
+│   ├── 1.txt
+│   ├── 2.txt
+│   └── n.txt
+├── combined.jsonl        # Combined JSONL content
+├── combined_with_metadata.jsonl
+├── combined.csv         # Combined CSV content
+└── combined_with_metadata.csv
+```
+
+## 📄 Output File Formats
+
+### 📝 Text Files (*.txt)
+
+The actual article content starts here. This is the clean, processed text of the article that was extracted from the webpage
+
+### 📑 Text Files with Metadata (texts_with_metadata/*.txt)
+
+title: My Awesome Page
+description: This is a great article about coding
+author: John Doe
+language: en
+dateScraped: 2024-01-20T10:30:00Z
+
+\-\-\-
+
+The actual article content starts here. This is the clean, processed text of the article that was extracted from the webpage.
+
+### 📊 JSONL Files (train.jsonl)
+
+```json
+{"text": "Clean article content here"}
+{"text": "Another article content here"}
+```
+
+### 📈 JSONL with Metadata (train_with_metadata.jsonl)
+
+```json
+{"text": "Article content", "metadata": {"title": "Page Title", "author": "John Doe"}}
+{"text": "Another article", "metadata": {"title": "Second Page", "author": "Jane Smith"}}
+```
+
+### 🗃️ JSON Files In Website Output  (*.json)
+
+```json
+{
+  "url": "<https://example.com/page>",
+  "title": "Page Title",
+  "description": "Page description",
+  "dateScraped": "2024-01-20T10:30:00Z"
+}
+```
+
+### 📋 CSV Files (train.csv)
+
+```csv
+text
+"Clean article content here"
+"Another article content here"
+```
+
+### 📊 CSV with Metadata (train_with_metadata.csv)
+
+```csv
+text,title,author,description
+"Article content","Page Title","John Doe","Page description"
+"Another article","Second Page","Jane Smith","Another description"
 ```
 
 ## 🤖 AI/LLM Training Ready

package/example-usage.js

@@ -68,5 +68,12 @@ void async function main ()
 
 	// 4
 	// https://electronicintifada.net/
+
+	// 5
+	// https://www.palestineremembered.com/ZionistFAQ.html
+
+	// 6 https://the-palestinian-side.vercel.app/
+
+	// 7 https://stand-with-palestine.org/blogs
 }()
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "clean-web-scraper",
-  "version": "3.2.2",
+  "version": "3.2.4",
   "main": "main.js",
   "scripts": {
     "start": "node main.js",

package/src/WebScraper.js

@@ -36,11 +36,11 @@ class WebScraper
 		this.excludeList = new Set( excludeList );
 		this.exactExcludeList = this.normalizeExcludeList( exactExcludeList );
 		this.allProcessedContent = [];
-		this.createOutputDirectory();
 	}
 
 	async start ()
 	{
+		this.createOutputDirectory();
 		await this.fetchPage( this.startURL, 0 );
 		this.createJSONLFile();
 		this.saveNumberedTextFiles();
@@ -384,41 +384,71 @@ class WebScraper
 		const fullOutputPath = path.join( __dirname, outputPath );
 
 		// Create output directories
+		WebScraper.createCombinedDirectories( fullOutputPath );
+
+		// Combine files by type
+		WebScraper.combineJSONLFiles( fullOutputPath, websites );
+		WebScraper.combineCSVFiles( fullOutputPath, websites );
+		WebScraper.combineTextFiles( fullOutputPath, websites );
+	}
+
+	static createCombinedDirectories ( fullOutputPath )
+	{
+		if ( fs.existsSync( fullOutputPath ) )
+		{
+			fs.rmSync( fullOutputPath, { recursive: true, force: true });
+		}
 		fs.mkdirSync( fullOutputPath, { recursive: true });
 		fs.mkdirSync( path.join( fullOutputPath, "texts" ), { recursive: true });
 		fs.mkdirSync( path.join( fullOutputPath, "texts_with_metadata" ), { recursive: true });
+	}
 
-		// Combine regular JSONL files
+	static combineJSONLFiles ( fullOutputPath, websites )
+	{
 		const jsonlOutput = fs.createWriteStream( path.join( fullOutputPath, "combined.jsonl" ) )
-		    .on( "error", ( err ) => { return console.error( "Error combining JSONL:", err ) });
+		.on( "error", ( err ) => { return console.error( "Error combining JSONL:", err ) });
 		const jsonlMetaOutput = fs.createWriteStream( path.join( fullOutputPath, "combined_with_metadata.jsonl" ) )
-		    .on( "error", ( err ) => { return console.error( "Error combining metadata JSONL:", err ) });
+		.on( "error", ( err ) => { return console.error( "Error combining metadata JSONL:", err ) });
+
+		for ( const website of websites )
+		{
+			const jsonlContent = fs.readFileSync( path.join( __dirname, website.jsonlOutputPath ), "utf-8" );
+			jsonlOutput.write( jsonlContent );
+
+			if ( website.includeMetadata )
+			{
+				const jsonlMetaContent = fs.readFileSync( path.join( __dirname, website.jsonlOutputPathWithMeta ), "utf-8" );
+				jsonlMetaOutput.write( jsonlMetaContent );
+			}
+		}
+
+		jsonlOutput.end();
+		jsonlMetaOutput.end();
+	}
+
+	static combineCSVFiles ( fullOutputPath, websites )
+	{
 		const csvOutput = fs.createWriteStream( path.join( fullOutputPath, "combined.csv" ) );
 		const csvMetaOutput = fs.createWriteStream( path.join( fullOutputPath, "combined_with_metadata.csv" ) );
 
 		csvOutput.write( "text\n" );
 		const metadataFields = websites.find( w => { return w.includeMetadata })?.metadataFields || new Set();
+
 		if ( metadataFields.size > 0 )
 		{
 			csvMetaOutput.write( `text,${Array.from( metadataFields ).join( "," )}\n` );
 		}
+
 		for ( const website of websites )
 		{
-			const jsonlContent = fs.readFileSync( path.join( __dirname, website.jsonlOutputPath ), "utf-8" );
-			jsonlOutput.write( jsonlContent );
-
 			const csvContent = fs.readFileSync( path.join( __dirname, website.csvOutputPath ), "utf-8" )
 			.split( "\n" )
 			.slice( 1 )
 			.filter( line => { return line.trim() });
 			csvOutput.write( `${csvContent.join( "\n" )}\n` );
 
-			// Combine metadata files if they exist
 			if ( website.includeMetadata )
 			{
-				const jsonlMetaContent = fs.readFileSync( path.join( __dirname, website.jsonlOutputPathWithMeta ), "utf-8" );
-				jsonlMetaOutput.write( jsonlMetaContent );
-
 				const csvMetaContent = fs.readFileSync( path.join( __dirname, website.csvOutputPathWithMeta ), "utf-8" )
 				.split( "\n" )
 				.slice( 1 )
@@ -427,18 +457,18 @@ class WebScraper
 			}
 		}
 
-		// Close all streams
-		jsonlOutput.end();
-		jsonlMetaOutput.end();
 		csvOutput.end();
 		csvMetaOutput.end();
+	}
 
-		// Combine text files (both regular and metadata versions)
+	static combineTextFiles ( fullOutputPath, websites )
+	{
 		let textFileCounter = 1;
+
 		for ( const website of websites )
 		{
-			// Regular text files
 			const textFiles = fs.readdirSync( path.join( __dirname, website.textOutputPath ) );
+
 			for ( const file of textFiles )
 			{
 				const content = fs.readFileSync( path.join( __dirname, website.textOutputPath, file ), "utf-8" );
@@ -448,7 +478,6 @@ class WebScraper
 					"utf-8"
 				);
 
-				// Metadata text files if they exist
 				if ( website.includeMetadata )
 				{
 					const metaContent = fs.readFileSync(
@@ -465,6 +494,7 @@ class WebScraper
 			}
 		}
 	}
+
 }
 
 module.exports = WebScraper;