easy-sitemap-generator 0.1.0 → 0.1.2

package/lib/sitemapGenerator.js

@@ -6,7 +6,7 @@ const path = require('path');
 const { escapeXml, normalizeUrl, calculatePriority } = require('../utils/xml.js');
 const { logInfo, logSuccess, logError, logWarning } = require('../utils/kleur.js');
 
-const VISITED_URLS = new Set();
+const VISITED_URLS = new Map();
 const IGNORED_PATTERNS = ['cdn-cgi', '?referrer=', '&referrer=', '/signin/v2/usernamerecovery', '/lifecycle/flows/signup', 'join?return_to='];
 const BASE_DELAY = 8000;
 
@@ -16,7 +16,6 @@ const delay = ms => new Promise(resolve => setTimeout(resolve, ms));
 const fetchUrl = async (url, retries = 0) => {
 	try {
 		logInfo(`GET ${url}`);
-
 		return await axios.get(url);
 	} catch (err) {
 		logError(`Error fetching URL: ${url} - ${err.message}`);
@@ -43,15 +42,13 @@ const fetchUrl = async (url, retries = 0) => {
 
 const crawl = async (url, baseUrl) => {
 	const normalizedUrl = normalizeUrl(url);
-	if (VISITED_URLS.has(normalizedUrl)) return; else VISITED_URLS.add(normalizedUrl);
+	if (VISITED_URLS.has(normalizedUrl)) return;
+	VISITED_URLS.set(normalizedUrl, { url: normalizedUrl });
 
-	const response = await fetchUrl(normalizedUrl);
-	if (!response) {
-		logWarning(`No response received for URL: ${normalizedUrl}`);
-		return;
-	}
+	const res = await fetchUrl(normalizedUrl);
+	if (!res) return logWarning(`No response received for URL: ${normalizedUrl}`);
 
-	const { document } = new JSDOM(response.data).window;
+	const { document } = new JSDOM(res.data).window;
 	const links = Array.from(document.querySelectorAll('a[href]'))
 		.map(link => urlModule.resolve(baseUrl, link.getAttribute('href')))
 		.map(normalizeUrl)
@@ -63,7 +60,11 @@ const crawl = async (url, baseUrl) => {
 		await crawl(link, baseUrl);
 	}
 
-	return { url: normalizedUrl, lastmod: response.headers['last-modified'] ? new Date(response.headers['last-modified']).toISOString() : new Date().toISOString() };
+	VISITED_URLS.set(normalizedUrl, {
+		url: normalizedUrl,
+		lastmod: (res.headers['last-modified'] ? new Date(res.headers['last-modified']) : new Date()).toISOString(),
+		priority: calculatePriority(normalizedUrl, baseUrl)
+	});
 };
 
 const generateSitemap = async (baseUrl, destination = 'sitemap.xml') => {
@@ -73,17 +74,11 @@ const generateSitemap = async (baseUrl, destination = 'sitemap.xml') => {
 
 	logInfo(`Generating sitemap with ${VISITED_URLS.size} URLs...`);
 
-	const urls = Array.from(VISITED_URLS)
-		.filter(url => shouldIncludeUrl(url, baseUrl))
-		.map(url => ({
-			url,
-			priority: calculatePriority(url, baseUrl),
-			lastmod: new Date().toISOString()
-		}))
+	const urls = Array.from(VISITED_URLS.values())
 		.sort((a, b) => b.priority - a.priority);
 
 	const sitemapContent = `<?xml version="1.0" encoding="UTF-8"?>
-<!-- Generated by sitemap generator - ${new Date()} -->
+<!-- Generated by https://github.com/sefinek24/easy-sitemap-generator at ${new Date().toISOString()} -->
 <urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://www.sitemaps.org/schemas/sitemap/0.9 http://www.sitemaps.org/schemas/sitemap/0.9/sitemap.xsd">
 ${urls.map(({ url, priority, lastmod }) => `    <url>
         <loc>${escapeXml(url)}</loc>

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "easy-sitemap-generator",
-  "version": "0.1.0",
+  "version": "0.1.2",
   "description": "Free and easy sitemap.xml file generator without any restrictions for your website. Try now!",
   "homepage": "https://github.com/sefinek24/easy-sitemap-generator#readme",
   "bugs": {
@@ -21,11 +21,11 @@
     "sitemap-generator": "./bin/cli.js"
   },
   "scripts": {
-    "test": "jest test/index.test.js",
+    "test": "echo \"Error: no test specified\" && exit 1",
     "up": "ncu -u && npm install && npm update && npm audit fix"
   },
   "dependencies": {
-    "axios": "^1.7.3",
+    "axios": "^1.7.4",
     "jsdom": "^24.1.1",
     "kleur": "^4.1.5"
   },