clean-web-scraper 3.2.0 → 3.2.2

package/README.md

@@ -11,10 +11,10 @@ A powerful Node.js web scraper that extracts clean, readable content from websit
 - 🚫 Excludes unwanted paths from scraping
 - 🔄 Handles relative and absolute URLs like a pro
 - 🎯 No duplicate page visits
-- 📊 Generates JSONL output file for ML training
-- 📊 AI-friendly clean text and csv output (perfect for LLM fine-tuning!)
+- 🤖 AI-friendly output formats (JSONL, CSV, clean text)
 - 📊 Rich metadata extraction
 - 📁 Combine results from multiple scrapers into a unified dataset
+- 🎯 Turn any website into an AI training dataset
 
 ## 🛠️ Prerequisites
 
@@ -57,16 +57,38 @@ const scraper = new WebScraper({
   includeMetadata: false,                       // Optional: Include metadata in output files
   metadataFields: ['title', 'description']      // Optional: Specify metadata fields to include
 });
-scraper.start();
-
-// Combine results from multiple scrapers
-WebScraper.combineResults('./combined-dataset', [scraper1, scraper2]);
+await scraper.start();
 ```
 
 ```bash
 node example-usage.js
 ```
 
+## 💻 Advanced Usage: Multi-Site Scraping
+
+```js
+const WebScraper = require('clean-web-scraper');
+
+// Scrape documentation website
+const docsScraper = new WebScraper({
+  baseURL: 'https://docs.example.com',
+  scrapResultPath: './datasets/docs'
+});
+
+// Scrape blog website
+const blogScraper = new WebScraper({
+  baseURL: 'https://blog.example.com',
+  scrapResultPath: './datasets/blog'
+});
+
+// Start scraping both sites
+await docsScraper.start();
+await blogScraper.start();
+
+// Combine all scraped content into a single dataset
+await WebScraper.combineResults('./combined-dataset', [docsScraper, blogScraper]);
+```
+
 ## 📤 Output
 
 Your AI-ready content is saved in a clean, structured format:
@@ -100,7 +122,7 @@ example.com/
 
 ## 🤖 AI/LLM Training Ready
 
-The output is specifically formatted for AI training purposes:
+The output is specifically formatted for AI training and fine-tuning purposes:
 
 - Clean, processed text without HTML markup
 - Multiple formats (JSONL, CSV, text files)

package/example-usage.js

@@ -22,7 +22,7 @@ async function khameneiIrFreePalestineTag ()
 		includeMetadata: true,
 		metadataFields: ["title", "description", "author", "lastModified", "language"]
 	});
-	// await scraper.start();
+	await scraper.start();
 	return scraper;
 }
 
@@ -50,7 +50,7 @@ async function decolonizepalestine ()
 		includeMetadata: true,
 		metadataFields: ["title", "description", "author", "lastModified", "language"]
 	});
-	// await scraper.start();
+	await scraper.start();
 	return scraper;
 }
 
@@ -58,8 +58,7 @@ void async function main ()
 {
 	const khameneiIrFreePalestineTagScraper = await khameneiIrFreePalestineTag();
 	const decolonizepalestineScraper = await decolonizepalestine();
-	await WebScraper.sleep( 1000 ); // Sleeps for 1 second
-	WebScraper.combineResults( "./dataset/combined", [
+	await WebScraper.combineResults( "./dataset/combined", [
 		khameneiIrFreePalestineTagScraper,
 		decolonizepalestineScraper
 	] );

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "clean-web-scraper",
-  "version": "3.2.0",
+  "version": "3.2.2",
   "main": "main.js",
   "scripts": {
     "start": "node main.js",

package/src/WebScraper.js

@@ -378,8 +378,9 @@ class WebScraper
 		return new Promise( resolve => { return setTimeout( resolve, ms ) });
 	}
 
-	static combineResults ( outputPath, websites )
+	static async combineResults ( outputPath, websites )
 	{
+		await WebScraper.sleep( 1000 );
 		const fullOutputPath = path.join( __dirname, outputPath );
 
 		// Create output directories