clean-web-scraper 2.0.1

package/.github/workflows/npm.yml

@@ -0,0 +1,19 @@
+name: Publish on NPM registry
+
+on:
+  push:
+    branches: ['main']
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-node@v2
+        with:
+          node-version: 20.x
+          registry-url: https://registry.npmjs.org/
+      # - run: npm install
+      - run: npm publish --access public
+        env:
+          NODE_AUTH_TOKEN: ${{secrets.NPM_TOKEN}}

package/README.md

@@ -0,0 +1,83 @@
+# 🕸️ Web Content Scraper
+
+A powerful Node.js web scraper that extracts clean, readable content from websites while keeping everything nicely organized. Perfect for creating AI training datasets! 🤖
+
+## ✨ Features
+
+- 🌐 Smart recursive web crawling of internal links
+- 📝 Clean content extraction using Mozilla's Readability
+- 🧹 Smart content processing and cleaning
+- 🗂️ Maintains original URL structure in saved files
+- 🚫 Excludes unwanted paths from scraping
+- 🔄 Handles relative and absolute URLs like a pro
+- 🎯 No duplicate page visits
+- 📊 Generates JSONL output file for ML training
+- 📊 AI-friendly clean text output (perfect for LLM fine-tuning!)
+
+## 🛠️ Prerequisites
+
+- Node.js (v18 or higher)
+- npm
+
+## 📦 Dependencies
+
+- **axios** - HTTP requests master
+- **jsdom** - DOM parsing wizard
+- **@mozilla/readability** - Content extraction genius
+
+## 🚀 Installation
+
+```bash
+npm i clean-web-scraper
+
+# OR
+
+git clone https://github.com/mlibre/Clean-Web-Scraper
+cd Clean-Web-Scraper
+npm install
+```
+
+## 💻 Usage
+
+```js
+const WebScraper = require('clean-web-scraper');
+
+const scraper = new WebScraper({
+  baseURL: 'https://example.com',      // Required: The website to scrape
+  folderPath: './output',              // Required: Where to save the content
+  excludeList: ['/admin', '/private'], // Optional: Paths to exclude
+  exactExcludeList: ['/specific-page'],// Optional: Exact URLs to exclude
+  jsonlPath: 'output.jsonl'            // Optional: Custom JSONL output path
+});
+
+scraper.start();
+```
+
+```bash
+node example-usage.js
+```
+
+## 📤 Output
+
+Your AI-ready content is saved in a clean, structured format:
+
+- 📁 Base folder: ./folderPath/example.com/
+- 📑 Files preserve original URL paths
+- 📝 Pure text format, perfect for LLM training and fine-tuning
+- 🤖 No HTML, no mess - just clean, structured text ready for AI consumption
+- 📊 JSONL output for ML training
+
+## 🤖 AI/LLM Training Ready
+
+The output is specifically formatted for AI training purposes:
+
+- Clean, processed text without HTML markup
+- Consistent formatting across all documents
+- Structured content perfect for fine-tuning LLMs
+- Ready to use in your ML pipelines
+
+## Standing with Palestine 🇵🇸
+
+This project supports Palestinian rights and stands in solidarity with Palestine. We believe in the importance of documenting and preserving Palestinian narratives, history, and struggles for justice and liberation.
+
+Free Palestine 🇵🇸

package/eslint.config.cjs

@@ -0,0 +1,64 @@
+// eslint.config.cjs
+
+module.exports = [
+	{
+		languageOptions: {
+			parserOptions: {
+				ecmaVersion: 13,
+				impliedStrict: true,
+			}
+		},
+		rules: {
+			"no-trailing-spaces": "error",
+			"linebreak-style": ["error", "unix"],
+			"quotes": ["error", "double"],
+			"one-var": ["error", "never"],
+			"brace-style": ["error", "allman", { allowSingleLine: true }],
+			"space-before-blocks": "warn",
+			"func-call-spacing": "error",
+			"space-before-function-paren": "error",
+			"space-in-parens": ["error", "always", { exceptions: ["{}"] }],
+			"keyword-spacing": "error",
+			"comma-spacing": "error",
+			"space-unary-ops": "error",
+			"block-spacing": "error",
+			"arrow-spacing": "error",
+			"key-spacing": "error",
+			"comma-style": "error",
+			"space-infix-ops": "error",
+			"array-bracket-spacing": "error",
+			"object-curly-spacing": ["error", "always"],
+			"no-multi-spaces": "error",
+			"operator-linebreak": "error",
+			"function-paren-newline": "warn",
+			"arrow-body-style": ["error", "always"],
+			"no-template-curly-in-string": "error",
+			"no-new-object": "error",
+			"no-extra-parens": ["error", "all", { conditionalAssign: false }],
+			"no-empty-function": "error",
+			"no-empty": ["warn", { allowEmptyCatch: true }],
+			"no-eq-null": "error",
+			"no-extra-bind": "error",
+			"no-self-compare": "error",
+			"no-useless-call": "error",
+			"no-undefined": "error",
+			"no-array-constructor": "error",
+			"prefer-destructuring": ["error",
+				{
+					VariableDeclarator: { array: false, object: true }, AssignmentExpression: { array: false, object: false } }, { enforceForRenamedProperties: false
+				}
+			],
+			"object-shorthand": "warn",
+			"prefer-spread": "warn",
+			"prefer-template": "warn",
+			"no-loop-func": "warn",
+			"prefer-rest-params": "warn",
+			"no-new-func": "warn",
+			"no-unneeded-ternary": "warn",
+			"no-process-exit": "off",
+			"require-await": "warn",
+			"indent": ["error", "tab", { MemberExpression: 0 }],
+			"no-tabs": 0,
+		},
+	},
+];

package/example-usage.js

@@ -0,0 +1,25 @@
+const WebScraper = require( "./src/WebScraper" );
+
+const baseURL = "https://decolonizepalestine.com";
+const folderPath = "./dataset";
+const excludeList = [
+	"https://decolonizepalestine.com/cdn-cgi",
+	"https://decolonizepalestine.com/introduction-to-palestine",
+	"https://decolonizepalestine.com/myths",
+	"https://decolonizepalestine.com/reading-list",
+	"https://decolonizepalestine.com/support-us"
+];
+const exactExcludeList = [
+	"https://decolonizepalestine.com/rainbow-washing",
+	"https://decolonizepalestine.com/"
+]
+
+
+const scraper = new WebScraper({
+	baseURL,
+	folderPath,
+	excludeList,
+	exactExcludeList,
+	jsonlPath: "./dataset/final.jsonl"
+});
+scraper.start();

package/main.js

@@ -0,0 +1,2 @@
+const WebScraper = require( "./src/WebScraper" );
+module.exports = WebScraper;

package/package.json

@@ -0,0 +1,30 @@
+{
+  "name": "clean-web-scraper",
+  "version": "2.0.1",
+  "main": "main.js",
+  "scripts": {
+    "start": "node main.js",
+    "test": "echo \"Error: no test specified\" && exit 1"
+  },
+  "keywords": [
+    "clean-web-scraper",
+    "web-scraper",
+    "scraper",
+    "scraper-js",
+    "scraper-js-library",
+    "web-scraper-js",
+    "ai-ready-web-scraper",
+    "ai",
+    "fine-tune",
+    "data-processing"
+  ],
+  "author": "",
+  "license": "ISC",
+  "description": "",
+  "dependencies": {
+    "@mozilla/readability": "^0.5.0",
+    "axios": "^1.7.9",
+    "eslint": "^9.17.0",
+    "jsdom": "^26.0.0"
+  }
+}

package/src/WebScraper.js

@@ -0,0 +1,208 @@
+const axios = require( "axios" );
+const jsdom = require( "jsdom" );
+const { JSDOM } = jsdom;
+const { Readability } = require( "@mozilla/readability" );
+const fs = require( "fs" );
+const path = require( "path" );
+
+class WebScraper
+{
+	constructor ({ baseURL, folderPath, excludeList, exactExcludeList, jsonlPath })
+	{
+		this.baseURL = baseURL;
+		this.jsonlPath = jsonlPath || "output.jsonl";
+		this.folderPath = path.join( folderPath, baseURL.replace( /^(https?:\/\/)?(www\.)?/, "" ).replace( /\/$/, "" ) );
+		this.visited = new Set();
+		this.excludeList = new Set( excludeList );
+		this.exactExcludeList = this.normalizeExcludeList( exactExcludeList );
+		this.processedContent = []; // Add this line
+		this.createOutputDirectory();
+	}
+
+	async start ()
+	{
+		this.visited.add( this.baseURL );
+		await this.fetchPage( this.baseURL );
+		this.createJSONLFile();
+	}
+
+	async fetchPage ( url )
+	{
+		try
+		{
+			const { data } = await axios.get( url );
+			const dom = new JSDOM( data, { url });
+
+			// Only save if the URL is not excluded
+			if ( !this.isExcluded( url ) )
+			{
+				const reader = new Readability( dom.window.document, { charThreshold: 500, nbTopCandidates: 20 });
+				const article = reader.parse();
+
+				if ( article )
+				{
+					this.saveArticle( url, article.textContent );
+				}
+				else
+				{
+					console.error( `No readable content found at ${url}` );
+				}
+			}
+
+			const links = this.extractLinks( data );
+			for ( const link of links )
+			{
+				if ( !this.visited.has( link ) )
+				{
+					this.visited.add( link );
+					await this.fetchPage( link );
+				}
+			}
+		}
+		catch ( error )
+		{
+			console.error( `Error fetching ${url}:`, error.message );
+		}
+	}
+
+	extractLinks ( data )
+	{
+		const links = new Set();
+		const regex = /<a\s+(?:[^>]*?\s+)?href=("|')(.*?)\1/gi;
+		let match;
+
+		while ( ( match = regex.exec( data ) ) !== null )
+		{
+			let href = match[2];
+			if ( href.endsWith( "/" ) )
+			{
+				href = href.slice( 0, -1 );
+			}
+			if ( href.startsWith( this.baseURL ) )
+			{
+				links.add( href );
+			}
+			else if ( href.startsWith( "/" ) )
+			{
+				links.add( new URL( href, this.baseURL ).href );
+			}
+		}
+
+		return links;
+	}
+
+	saveArticle ( url, content )
+	{
+		const processedContent = this.processContent( content );
+
+		this.processedContent.push({
+			text: processedContent.trim()
+		});
+
+		let urlPath = new URL( url ).pathname;
+		if ( urlPath === "/" )
+		{
+			urlPath = "/index";
+		}
+		const filePath = path.join( __dirname, this.folderPath, urlPath );
+		const dir = path.dirname( filePath );
+
+		// Create metadata object
+		const metadata = {
+			url,
+			dateScraped: new Date().toISOString(),
+			contentLength: processedContent.length,
+			fileName: `${path.basename( filePath )}.txt`
+		};
+
+		// Create directory if it doesn't exist
+		fs.mkdirSync( dir, { recursive: true });
+
+		// Save the text content
+		fs.writeFileSync( `${filePath}.txt`, processedContent, "utf-8" );
+
+		// Save the JSON metadata
+		fs.writeFileSync( `${filePath}.json`, JSON.stringify( metadata, null, 2 ), "utf-8" );
+
+		console.log( `Saved: ${filePath}.txt` );
+		console.log( `Saved: ${filePath}.json` );
+	}
+
+	createJSONLFile ()
+	{
+		const writeStream = fs.createWriteStream( path.join( __dirname, this.jsonlPath ) );
+
+		for ( const content of this.processedContent )
+		{
+			const jsonLine = `${JSON.stringify( content )}\n`;
+			writeStream.write( jsonLine );
+		}
+
+		writeStream.end();
+		console.log( `Created JSONL file at: ${this.jsonlPath}` );
+	}
+
+	processContent ( content )
+	{
+		let processed = content;
+
+		// Remove "[You can read more about this here]" and similar patterns
+		processed = processed.replace( /\[You can read more about this here\]/g, "" ).trim();
+
+		// Trim each line
+		processed = processed.split( "\n" )
+		.map( line => { return line.trim() })
+		.join( "\n" );
+
+		// Replace 3 or more newlines with a single newline
+		processed = processed.replace( /\n{3,}/g, "\n\n" );
+
+		// Add more processing rules as needed:
+		// processed = processed.replace(/\[.*?\]/g, ''); // Removes all content within square brackets
+		// processed = processed.replace(/\(.*?\)/g, ''); // Removes all content within parentheses
+
+		return processed;
+	}
+
+	normalizeExcludeList ( list )
+	{
+		const normalizedSet = new Set();
+		for ( let i = 0; i < list.length; i++ )
+		{
+			const item = list[i];
+			if ( item.endsWith( "/" ) )
+			{
+				normalizedSet.add( item.slice( 0, -1 ) );
+			}
+			else
+			{
+				normalizedSet.add( item );
+			}
+			normalizedSet.add( `${item.endsWith( "/" ) ? item : `${item }/`}` );
+		}
+		return normalizedSet;
+	}
+
+	isExcluded ( url )
+	{
+		if ( this.exactExcludeList.has( url ) )
+		{
+			return true;
+		}
+		return Array.from( this.excludeList ).some( excluded => { return url.startsWith( excluded ) });
+	}
+
+	createOutputDirectory ()
+	{
+		if ( fs.existsSync( path.join( __dirname, this.folderPath ) ) )
+		{
+			fs.rmSync( path.join( __dirname, this.folderPath ), { recursive: true, force: true });
+		}
+		if ( !fs.existsSync( path.join( __dirname, this.folderPath ) ) )
+		{
+			fs.mkdirSync( path.join( __dirname, this.folderPath ), { recursive: true });
+		}
+	}
+}
+
+module.exports = WebScraper;