clean-web-scraper 2.1.0 → 2.2.1

package/README.md

@@ -11,8 +11,8 @@ A powerful Node.js web scraper that extracts clean, readable content from websit
 - 🚫 Excludes unwanted paths from scraping
 - 🔄 Handles relative and absolute URLs like a pro
 - 🎯 No duplicate page visits
-- 📊 Generates JSONL and raw text output file for ML training
-- 📊 AI-friendly clean text output (perfect for LLM fine-tuning!)
+- 📊 Generates JSONL output file for ML training
+- 📊 AI-friendly clean text and csv output (perfect for LLM fine-tuning!)
 
 ## 🛠️ Prerequisites
 
@@ -44,11 +44,12 @@ const WebScraper = require('clean-web-scraper');
 
 const scraper = new WebScraper({
   baseURL: 'https://example.com',       // Required: The website to scrape
-  scrapResultPath: './output',          // Required: Where to save the content
   excludeList: ['/admin', '/private'],  // Optional: Paths to exclude
   exactExcludeList: ['/specific-page'], // Optional: Exact URLs to exclude
-  jsonlPath: 'output.jsonl',            // Optional: Custom JSONL output path
-  textOutputPath: "./dataset/texts"     // Optional: Custom text output path
+  scrapResultPath: './dataset',         // Required: Where to save the content
+  jsonlPath: './dataset/train.jsonl',   // Optional: Custom JSONL output path
+  textOutputPath: "./dataset/texts",    // Optional: Custom text output path
+  csvPath: "./dataset/train.csv"        // Optional: Custom CSV output path
 });
 
 scraper.start();
@@ -67,6 +68,31 @@ Your AI-ready content is saved in a clean, structured format:
 - 📝 Pure text format, perfect for LLM training and fine-tuning
 - 🤖 No HTML, no mess - just clean, structured text ready for AI consumption
 - 📊 JSONL output for ML training
+- 📈 CSV output with clean text content
+
+```bash
+dataset/
+├── decolonizepalestine.com
+│   ├── faq.json
+│   ├── faq.txt
+│   ├── intro
+│   │   ├── bds-101.json
+│   │   ├── bds-101.txt
+│   ├── myth
+│   │   ├── a-land-without-a-people-for-a-people-without-a-land.json
+│   │   ├── a-land-without-a-people-for-a-people-without-a-land.txt
+│   └── rainbow-washing
+│       ├── bluewashing.json
+│       ├── bluewashing.txt
+├── texts
+│   ├── 1.txt
+│   ├── 2.txt
+│   ├── 3.txt
+│   ├── 4.txt
+│   └── 5.txt
+├── train.csv
+└── train.jsonl
+```
 
 ## 🤖 AI/LLM Training Ready
 

package/example-usage.js

@@ -22,6 +22,7 @@ const scraper = new WebScraper({
 	excludeList,
 	exactExcludeList,
 	jsonlPath: "./dataset/train.jsonl",
-	textOutputPath: "./dataset/texts"
+	textOutputPath: "./dataset/texts",
+	csvPath: "./dataset/train.csv"
 });
 scraper.start();

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "clean-web-scraper",
-  "version": "2.1.0",
+  "version": "2.2.1",
   "main": "main.js",
   "scripts": {
     "start": "node main.js",

package/src/WebScraper.js

@@ -13,13 +13,15 @@ class WebScraper
 		exactExcludeList,
 		scrapResultPath = "./dataset",
 		jsonlPath,
-		textOutputPath
+		textOutputPath,
+		csvPath
 	})
 	{
 		this.baseURL = baseURL;
 		this.scrapResultPath = path.join( scrapResultPath, baseURL.replace( /^(https?:\/\/)?(www\.)?/, "" ).replace( /\/$/, "" ) );
 		this.jsonlPath = jsonlPath || path.join( this.scrapResultPath, "train.jsonl" );
 		this.textOutputPath = textOutputPath || path.join( this.scrapResultPath, "texts" );
+		this.csvPath = csvPath || path.join( this.scrapResultPath, "train.csv" );
 		this.visited = new Set();
 		this.excludeList = new Set( excludeList );
 		this.exactExcludeList = this.normalizeExcludeList( exactExcludeList );
@@ -33,6 +35,7 @@ class WebScraper
 		await this.fetchPage( this.baseURL );
 		this.createJSONLFile();
 		this.saveNumberedTextFiles();
+		this.createCSVFile();
 		console.log( "Scraping completed." );
 	}
 
@@ -152,6 +155,23 @@ class WebScraper
 		console.log( `Created JSONL file at: ${this.jsonlPath}` );
 	}
 
+	createCSVFile ()
+	{
+		const writeStream = fs.createWriteStream( path.join( __dirname, this.csvPath ) );
+
+		writeStream.write( "text\n" );
+
+		for ( const content of this.processedContent )
+		{
+			const escapedText = content.text.replace( /"/g, "\"\"" );
+			const csvLine = `"${escapedText}"\n`;
+			writeStream.write( csvLine );
+		}
+
+		writeStream.end();
+		console.log( `Created CSV file at: ${this.csvPath}` );
+	}
+
 	saveNumberedTextFiles ()
 	{
 		this.processedContent.forEach( ( content, index ) =>