easy-sitemap-generator 0.1.0

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2024 Sefinek
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/README.md

@@ -0,0 +1,44 @@
+# 🗺️ Easy Sitemap.xml generator without any limits
+Finally! A free and easy-to-use `sitemap.xml` generator with no restrictions for your website.
+Improve your search engine rankings effortlessly! All you need is Node.js installed and this module. Good luck!
+
+<a href="https://www.npmjs.com/package/easy-sitemap-generator" target="_blank" title="easy-sitemap-generator - npm" style="text-decoration:none">
+    <img src="https://img.shields.io/npm/dt/easy-sitemap-generator.svg?maxAge=3600" alt="The number of downloads">
+    <img src="https://img.shields.io/github/issues/sefinek24/easy-sitemap-generator" alt="Issues">
+    <img src="https://img.shields.io/github/last-commit/sefinek24/easy-sitemap-generator" alt="Last commit">
+    <img src="https://img.shields.io/github/commit-activity/w/sefinek24/easy-sitemap-generator" alt="Commit activity">
+    <img src="https://img.shields.io/github/languages/code-size/sefinek24/easy-sitemap-generator" alt="Code size">
+</a>
+
+## 🤔 How to use it?
+### CLI (recommenced)
+```bash
+npm install easy-sitemap-generator -g
+sitemap --domain=example.com
+```
+#### Aliases
+| sitemap-gen | sitemap-generator | generate-sitemap |
+|-------------|-------------------|------------------|
+
+### Script
+```js
+const sitemap = require('easy-sitemap-generator');
+
+(async () => {
+   const content = await sitemap.generate('https://example.com');
+   console.log(content);
+})();
+```
+
+## ✔️ Sample generated file
+https://sefinek.net/sitemap.xml
+
+## 👀 Why do I need this?
+Indexing bots, such as Google, often check the sitemap.xml file by making a `GET /sitemap.xml` request to find subpages of your website.
+This can improve your site’s visibility in search engine results. Sitemap files are a standard feature and can be found on every web server.
+
+## 😉 Important
+Before running the script or executing the `sitemap` CLI command, make sure you have a stable internet connection. Also, disconnect from any proxy or VPN if you're connected.
+
+## 📘 License
+Licensed under the MIT License. See the [LICENSE](LICENSE) file for more details.

package/bin/cli.js

@@ -0,0 +1,16 @@
+#!/usr/bin/env node
+
+const { logError } = require('../utils/kleur.js');
+const { generate } = require('../lib/sitemapGenerator');
+
+const args = process.argv.slice(2);
+const urlArg = args.find(arg => arg.startsWith('--domain='));
+if (!urlArg) {
+	logError('No URL provided. Use: sitemap-generator --domain=<YOUR-DOMAIN>');
+	process.exit(1);
+}
+
+generate(`https://${urlArg.split('=')[1].replace(/(^\w+:|^)\/\//, '')}`).catch(err => {
+	logError(err);
+	process.exit(2);
+});

package/index.d.ts

@@ -0,0 +1,16 @@
+declare module 'easy-sitemap-generator' {
+    /**
+     * Generates a sitemap for the specified webpage and saves it to the given destination.
+     *
+     * @param url - The URL of the webpage for which the sitemap is to be generated.
+     * @param destination - Optional. The path to the file where the generated sitemap will be saved.
+     * If not specified, the sitemap will be saved in the same folder where the script is run (default: './sitemap.xml').
+     * @returns A promise that resolves to a string containing the contents of the generated sitemap.xml file.
+     */
+    export function generate(url: string, destination?: string): Promise<string>;
+
+    /**
+     * The current version of the `easy-sitemap-generator` module.
+     */
+    export const version: string;
+}

package/index.js

@@ -0,0 +1,6 @@
+const { generate, version } = require('./lib/sitemapGenerator.js');
+
+module.exports = {
+	generate,
+	version
+};

package/lib/sitemapGenerator.js

@@ -0,0 +1,105 @@
+const { JSDOM } = require('jsdom');
+const { axios, version } = require('../services/axios.js');
+const urlModule = require('url');
+const fs = require('fs/promises');
+const path = require('path');
+const { escapeXml, normalizeUrl, calculatePriority } = require('../utils/xml.js');
+const { logInfo, logSuccess, logError, logWarning } = require('../utils/kleur.js');
+
+const VISITED_URLS = new Set();
+const IGNORED_PATTERNS = ['cdn-cgi', '?referrer=', '&referrer=', '/signin/v2/usernamerecovery', '/lifecycle/flows/signup', 'join?return_to='];
+const BASE_DELAY = 8000;
+
+const shouldIncludeUrl = (url, baseUrl) => !IGNORED_PATTERNS.some(pattern => url.includes(pattern)) && url.startsWith(baseUrl);
+const delay = ms => new Promise(resolve => setTimeout(resolve, ms));
+
+const fetchUrl = async (url, retries = 0) => {
+	try {
+		logInfo(`GET ${url}`);
+
+		return await axios.get(url);
+	} catch (err) {
+		logError(`Error fetching URL: ${url} - ${err.message}`);
+		if (err.response) {
+			const statusCode = err.response.status;
+			if (statusCode === 429) {
+				const delayTime = BASE_DELAY * Math.pow(2, retries);
+				logWarning(`Rate limit hit. Retrying in ${(delayTime / 1000).toFixed(2)}s... (Attempt ${retries + 1})`);
+				await delay(delayTime);
+				return fetchUrl(url, retries + 1);
+			} else if (statusCode >= 500) {
+				logError(`Failed to fetch ${url}. Status code: ${statusCode}. Skipping...`);
+				return null;
+			} else if (statusCode >= 400) {
+				logWarning(`Failed to fetch ${url}. Status code: ${statusCode}. Skipping...`);
+				return null;
+			}
+		} else {
+			logError(`Failed to fetch ${url}. Unknown error: ${err.message}. Skipping...`);
+			return null;
+		}
+	}
+};
+
+const crawl = async (url, baseUrl) => {
+	const normalizedUrl = normalizeUrl(url);
+	if (VISITED_URLS.has(normalizedUrl)) return; else VISITED_URLS.add(normalizedUrl);
+
+	const response = await fetchUrl(normalizedUrl);
+	if (!response) {
+		logWarning(`No response received for URL: ${normalizedUrl}`);
+		return;
+	}
+
+	const { document } = new JSDOM(response.data).window;
+	const links = Array.from(document.querySelectorAll('a[href]'))
+		.map(link => urlModule.resolve(baseUrl, link.getAttribute('href')))
+		.map(normalizeUrl)
+		.filter(link => shouldIncludeUrl(link, baseUrl));
+
+	logInfo(`Found ${links.length} urls on ${normalizedUrl}`);
+
+	for (const link of links) {
+		await crawl(link, baseUrl);
+	}
+
+	return { url: normalizedUrl, lastmod: response.headers['last-modified'] ? new Date(response.headers['last-modified']).toISOString() : new Date().toISOString() };
+};
+
+const generateSitemap = async (baseUrl, destination = 'sitemap.xml') => {
+	logInfo(`Starting crawl for base URL: ${baseUrl}`);
+
+	await crawl(baseUrl, baseUrl);
+
+	logInfo(`Generating sitemap with ${VISITED_URLS.size} URLs...`);
+
+	const urls = Array.from(VISITED_URLS)
+		.filter(url => shouldIncludeUrl(url, baseUrl))
+		.map(url => ({
+			url,
+			priority: calculatePriority(url, baseUrl),
+			lastmod: new Date().toISOString()
+		}))
+		.sort((a, b) => b.priority - a.priority);
+
+	const sitemapContent = `<?xml version="1.0" encoding="UTF-8"?>
+<!-- Generated by sitemap generator - ${new Date()} -->
+<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://www.sitemaps.org/schemas/sitemap/0.9 http://www.sitemaps.org/schemas/sitemap/0.9/sitemap.xsd">
+${urls.map(({ url, priority, lastmod }) => `    <url>
+        <loc>${escapeXml(url)}</loc>
+        <lastmod>${lastmod}</lastmod>
+        <priority>${priority.toFixed(2)}</priority>
+    </url>`).join('\n')}
+</urlset>`;
+
+	const output = path.resolve(destination);
+	await fs.writeFile(output, sitemapContent, 'utf8');
+	logSuccess(`Sitemap has been generated at ${output}`);
+
+	return sitemapContent;
+};
+
+module.exports = {
+	generate: generateSitemap,
+	version
+};

package/package.json

@@ -0,0 +1,36 @@
+{
+  "name": "easy-sitemap-generator",
+  "version": "0.1.0",
+  "description": "Free and easy sitemap.xml file generator without any restrictions for your website. Try now!",
+  "homepage": "https://github.com/sefinek24/easy-sitemap-generator#readme",
+  "bugs": {
+    "url": "https://github.com/sefinek24/easy-sitemap-generator/issues"
+  },
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/sefinek24/easy-sitemap-generator.git"
+  },
+  "license": "MIT",
+  "author": "Sefinek <contact@nekosia.cat> (https://sefinek.net)",
+  "main": "index.js",
+  "typings": "index.d.ts",
+  "bin": {
+    "generate-sitemap": "./bin/cli.js",
+    "sitemap": "./bin/cli.js",
+    "sitemap-gen": "./bin/cli.js",
+    "sitemap-generator": "./bin/cli.js"
+  },
+  "scripts": {
+    "test": "jest test/index.test.js",
+    "up": "ncu -u && npm install && npm update && npm audit fix"
+  },
+  "dependencies": {
+    "axios": "^1.7.3",
+    "jsdom": "^24.1.1",
+    "kleur": "^4.1.5"
+  },
+  "devDependencies": {
+    "@eslint/js": "^9.9.0",
+    "globals": "^15.9.0"
+  }
+}

package/services/axios.js

@@ -0,0 +1,7 @@
+const axios = require('axios');
+const { version } = require('../package.json');
+
+axios.defaults.timeout = 24000;
+axios.defaults.headers.common['User-Agent'] = `Mozilla/5.0 (compatible; EasySitemapGen/${version}; +https://github.com/sefinek24/easy-sitemap-generator)`;
+
+module.exports = { axios, version };

package/utils/kleur.js

@@ -0,0 +1,8 @@
+const kleur = require('kleur');
+
+const logInfo = msg => console.log(kleur.blue().bold('[INFO]:'), msg);
+const logSuccess = msg => console.log(kleur.green().bold('[SUCCESS]:'), msg);
+const logError = msg => console.error(kleur.red().bold('[ERROR]:'), msg);
+const logWarning = msg => console.warn(kleur.yellow().bold('[WARN]:'), msg);
+
+module.exports = { logInfo, logSuccess, logError, logWarning };

package/utils/xml.js

@@ -0,0 +1,27 @@
+const escapeXml = str =>
+	str.replace(/&/g, '&')
+		.replace(/</g, '&lt;')
+		.replace(/>/g, '&gt;')
+		.replace(/"/g, '&quot;')
+		.replace(/'/g, '&apos;');
+
+const normalizeUrl = url => {
+	const parsedUrl = new URL(url);
+	parsedUrl.hash = '';
+	return parsedUrl.toString();
+};
+
+const calculatePriority = (url, baseUrl) => {
+	const path = url.replace(baseUrl, '').split('/').filter(Boolean);
+	const depth = path.length;
+	const hasQuery = url.includes('?');
+
+	if (depth === 0) return 1.0;
+	if (depth === 1) return 0.81;
+	if (depth === 2) return hasQuery ? 0.51 : 0.71;
+	if (depth >= 3) return hasQuery ? 0.31 : 0.41;
+
+	return 0.5;
+};
+
+module.exports = { escapeXml, normalizeUrl, calculatePriority };