easy-sitemap-generator 0.1.2 → 0.1.4

package/lib/sitemapGenerator.js

@@ -1,14 +1,14 @@
 const { JSDOM } = require('jsdom');
-const { axios, version } = require('../services/axios.js');
 const urlModule = require('url');
 const fs = require('fs/promises');
 const path = require('path');
+const { axios, version } = require('../services/axios.js');
 const { escapeXml, normalizeUrl, calculatePriority } = require('../utils/xml.js');
 const { logInfo, logSuccess, logError, logWarning } = require('../utils/kleur.js');
 
 const VISITED_URLS = new Map();
 const IGNORED_PATTERNS = ['cdn-cgi', '?referrer=', '&referrer=', '/signin/v2/usernamerecovery', '/lifecycle/flows/signup', 'join?return_to='];
-const BASE_DELAY = 8000;
+const BASE_DELAY = 9000;
 
 const shouldIncludeUrl = (url, baseUrl) => !IGNORED_PATTERNS.some(pattern => url.includes(pattern)) && url.startsWith(baseUrl);
 const delay = ms => new Promise(resolve => setTimeout(resolve, ms));
@@ -16,21 +16,27 @@ const delay = ms => new Promise(resolve => setTimeout(resolve, ms));
 const fetchUrl = async (url, retries = 0) => {
 	try {
 		logInfo(`GET ${url}`);
-		return await axios.get(url);
+
+		const res = await axios.get(url);
+		if (res.status === 200) {
+			return res;
+		} else {
+			logWarning(`Non-200 status code (${res.status}) for URL: ${url}. Skipping...`);
+			return null;
+		}
 	} catch (err) {
-		logError(`Error fetching URL: ${url} - ${err.message}`);
 		if (err.response) {
 			const statusCode = err.response.status;
 			if (statusCode === 429) {
 				const delayTime = BASE_DELAY * Math.pow(2, retries);
-				logWarning(`Rate limit hit. Retrying in ${(delayTime / 1000).toFixed(2)}s... (Attempt ${retries + 1})`);
+				logWarning(`429: Rate limit hit! Retrying in ${(delayTime / 1000).toFixed(2)}s... (Attempt ${retries + 1})`);
 				await delay(delayTime);
 				return fetchUrl(url, retries + 1);
-			} else if (statusCode >= 500) {
-				logError(`Failed to fetch ${url}. Status code: ${statusCode}. Skipping...`);
+			} else if (statusCode === 404) {
+				logWarning('404: Not Found');
 				return null;
-			} else if (statusCode >= 400) {
-				logWarning(`Failed to fetch ${url}. Status code: ${statusCode}. Skipping...`);
+			} else {
+				logError(`${statusCode}: Failed to fetch! Skipping...`);
 				return null;
 			}
 		} else {
@@ -43,10 +49,11 @@ const fetchUrl = async (url, retries = 0) => {
 const crawl = async (url, baseUrl) => {
 	const normalizedUrl = normalizeUrl(url);
 	if (VISITED_URLS.has(normalizedUrl)) return;
-	VISITED_URLS.set(normalizedUrl, { url: normalizedUrl });
 
 	const res = await fetchUrl(normalizedUrl);
-	if (!res) return logWarning(`No response received for URL: ${normalizedUrl}`);
+	if (!res) return;
+
+	VISITED_URLS.set(normalizedUrl, { url: normalizedUrl });
 
 	const { document } = new JSDOM(res.data).window;
 	const links = Array.from(document.querySelectorAll('a[href]'))
@@ -54,7 +61,7 @@ const crawl = async (url, baseUrl) => {
 		.map(normalizeUrl)
 		.filter(link => shouldIncludeUrl(link, baseUrl));
 
-	logInfo(`Found ${links.length} urls on ${normalizedUrl}`);
+	logInfo(`${res.status}: Found ${links.length} urls`);
 
 	for (const link of links) {
 		await crawl(link, baseUrl);
@@ -73,17 +80,18 @@ const generateSitemap = async (baseUrl, destination = 'sitemap.xml') => {
 	await crawl(baseUrl, baseUrl);
 
 	logInfo(`Generating sitemap with ${VISITED_URLS.size} URLs...`);
+	// console.log(VISITED_URLS);
 
 	const urls = Array.from(VISITED_URLS.values())
 		.sort((a, b) => b.priority - a.priority);
 
 	const sitemapContent = `<?xml version="1.0" encoding="UTF-8"?>
-<!-- Generated by https://github.com/sefinek24/easy-sitemap-generator at ${new Date().toISOString()} -->
+<!-- Generated by https://github.com/sefinek24/easy-sitemap-generator v${version} at ${new Date().toISOString()} -->
 <urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://www.sitemaps.org/schemas/sitemap/0.9 http://www.sitemaps.org/schemas/sitemap/0.9/sitemap.xsd">
 ${urls.map(({ url, priority, lastmod }) => `    <url>
         <loc>${escapeXml(url)}</loc>
         <lastmod>${lastmod}</lastmod>
-        <priority>${priority.toFixed(2)}</priority>
+        <priority>${priority?.toFixed(2) || 0.50}</priority>
     </url>`).join('\n')}
 </urlset>`;
 

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "easy-sitemap-generator",
-  "version": "0.1.2",
-  "description": "Free and easy sitemap.xml file generator without any restrictions for your website. Try now!",
+  "version": "0.1.4",
+  "description": "Easy and free sitemap.xml file generator without any restrictions for your website.",
   "homepage": "https://github.com/sefinek24/easy-sitemap-generator#readme",
   "bugs": {
     "url": "https://github.com/sefinek24/easy-sitemap-generator/issues"
@@ -25,12 +25,12 @@
     "up": "ncu -u && npm install && npm update && npm audit fix"
   },
   "dependencies": {
-    "axios": "^1.7.4",
-    "jsdom": "^24.1.1",
+    "axios": "^1.7.5",
+    "jsdom": "^25.0.0",
     "kleur": "^4.1.5"
   },
   "devDependencies": {
-    "@eslint/js": "^9.9.0",
+    "@eslint/js": "^9.9.1",
     "globals": "^15.9.0"
   }
 }

package/utils/kleur.js

@@ -1,8 +1,8 @@
 const kleur = require('kleur');
 
-const logInfo = msg => console.log(kleur.blue().bold('[INFO]:'), msg);
-const logSuccess = msg => console.log(kleur.green().bold('[SUCCESS]:'), msg);
-const logError = msg => console.error(kleur.red().bold('[ERROR]:'), msg);
-const logWarning = msg => console.warn(kleur.yellow().bold('[WARN]:'), msg);
+const logInfo = msg => console.log(kleur.blue().bold('[INFO]: ') + msg);
+const logSuccess = msg => console.log(kleur.green().bold('[SUCCESS]: ') + msg);
+const logError = msg => console.error(kleur.red().bold('[ERROR]: ') + msg);
+const logWarning = msg => console.warn(kleur.yellow().bold('[WARN]: ') + msg);
 
 module.exports = { logInfo, logSuccess, logError, logWarning };

package/utils/xml.js

@@ -17,9 +17,9 @@ const calculatePriority = (url, baseUrl) => {
 	const hasQuery = url.includes('?');
 
 	if (depth === 0) return 1.0;
-	if (depth === 1) return 0.81;
-	if (depth === 2) return hasQuery ? 0.51 : 0.71;
-	if (depth >= 3) return hasQuery ? 0.31 : 0.41;
+	if (depth === 1) return 0.85;
+	if (depth === 2) return hasQuery ? 0.54 : 0.74;
+	if (depth >= 3) return hasQuery ? 0.34 : 0.44;
 
 	return 0.5;
 };