@cocreate/sitemap 1.1.0 → 1.2.1

package/CHANGELOG.md

@@ -1,3 +1,24 @@
+## [1.2.1](https://github.com/CoCreate-app/CoCreate-sitemap/compare/v1.2.0...v1.2.1) (2024-09-01)
+
+
+### Bug Fixes
+
+* return if no title, parse sub object in createEntry ([97d8751](https://github.com/CoCreate-app/CoCreate-sitemap/commit/97d8751808e9f7e43c61de864820905949f89dba))
+
+# [1.2.0](https://github.com/CoCreate-app/CoCreate-sitemap/compare/v1.1.0...v1.2.0) (2024-08-30)
+
+
+### Bug Fixes
+
+* add sitemap to sitemapindex if not already in index ([0ff6911](https://github.com/CoCreate-app/CoCreate-sitemap/commit/0ff6911fe81eeb05295639fb984401d3e76b90ad))
+* noindex metatag return and handling sitemap generation ([058116a](https://github.com/CoCreate-app/CoCreate-sitemap/commit/058116a34c44100bfa4ea1b20afe13fd489bc642))
+
+
+### Features
+
+* added node-html-parser ([397c590](https://github.com/CoCreate-app/CoCreate-sitemap/commit/397c5908af1149d2d60d7c431a8a0f1bd60789f4))
+* parseHtml function to read and generate nested sitemaps using sitemap-* attributes and metatags ([64edbdb](https://github.com/CoCreate-app/CoCreate-sitemap/commit/64edbdbdcee6f64bc5080997557f9691f072381c))
+
 # [1.1.0](https://github.com/CoCreate-app/CoCreate-sitemap/compare/v1.0.0...v1.1.0) (2024-08-24)
 
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@cocreate/sitemap",
-  "version": "1.1.0",
+  "version": "1.2.1",
   "description": "A simple sitemap component in vanilla javascript. Easily configured using HTML5 data-attributes and/or JavaScript API.",
   "keywords": [
     "sitemap",
@@ -43,5 +43,8 @@
     "type": "GitHub Sponsors ❤",
     "url": "https://github.com/sponsors/CoCreate-app"
   },
-  "main": "./src/index.js"
+  "main": "./src/index.js",
+  "dependencies": {
+    "node-html-parser": "^6.1.13"
+  }
 }

package/src/index.js

@@ -20,6 +20,8 @@
 // you must obtain a commercial license from CoCreate LLC.
 // For details, visit <https://cocreate.app/licenses/> or contact us at sales@cocreate.app.
 
+const { parse } = require("node-html-parser");
+
 class CoCreateSitemap {
     constructor(crud) {
         this.crud = crud;
@@ -34,14 +36,18 @@ class CoCreateSitemap {
         if (!file.public || file.public === "false")
             return;
 
-        // Check if the file is HTML and contains a noindex meta tag
-        if (file['content-type'] === 'text/html' && file.src.includes('<meta name="robots" content="noindex">'))
-            return;
+        // Check if the file is HTML and contains a noindex meta or title tag
+        if (file['content-type'] === 'text/html') {
+            if (/<meta\s+name=["']robots["']\s+content=["'][^"']*noindex[^"']*["']/i.test(file.src))
+                return;
+            if (!(/<title[^>]*>[\s\S]*?<\/title>/i.test(file.src)))
+                return;
+        }
 
         // Compare the lastmod date in the sitemap with the modified.on date
         if (file.sitemap && file.sitemap.lastmod && file.modified.on) {
-            if (new Date(file.sitemap.lastmod) >= new Date(file.modified.on))
-                return;
+            // if (new Date(file.sitemap.lastmod) >= new Date(file.modified.on))
+            //     return;
         }
 
         // Logic to update the sitemap
@@ -50,14 +56,17 @@ class CoCreateSitemap {
 
     async updateSitemap(file, host) {
         try {
+            if (!file.sitemap)
+                file.sitemap = {}
+
             // TODO: need to get info such as host
             const entry = this.createEntry(file);
 
             let { mainSitemap, sitemap } = await this.getSitemap(file, host);
 
-            if (file.sitemap.pathname) {
+            if (file.pathname) {
                 // Perform regex search starting at the pathname
-                const regexPattern = `<url>\\s*<loc>.*?${file.sitemap.pathname.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}.*?</loc>[\\s\\S]*?</url>`;
+                const regexPattern = `<url>\\s*<loc>.*?${file.pathname.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}.*?</loc>[\\s\\S]*?</url>`;
                 const match = sitemap.src.match(new RegExp(regexPattern));
 
                 if (match) {
@@ -85,35 +94,54 @@ class CoCreateSitemap {
     }
 
     createEntry(file) {
-        const depth = (file.pathname.match(/\//g) || []).length;
-        const priority = Math.max(0.1, 1.0 - (depth - 1) * 0.1).toFixed(1);
-
-        const defaultKeys = {
-            loc: file.pathname,
-            lastmod: file.modified.on,
-            changefreq: 'monthly', // Example default value
-            priority: priority,
-        };
-        // Merge default keys with file.sitemap, prioritizing file.sitemap values
-        file.sitemap = { ...defaultKeys, ...file.sitemap };
+        file.sitemap.loc = file.pathname;
         file.sitemap.lastmod = file.modified.on;
 
+        if (file['content-type'] === 'text/html') {
+            this.parseHtml(file)
+
+            if (file.sitemap.type !== 'news') {
+                if (file.sitemap.changefreq)
+                    file.sitemap.changefreq = 'monthly';
+
+                if (!file.sitemap.priority) {
+                    const depth = (file.pathname.match(/\//g) || []).length;
+                    file.sitemap.priority = Math.max(0.1, 1.0 - (depth - 1) * 0.1).toFixed(1);
+                }
+            }
+        }
+
         let entry = `\t<url>\n`;
 
         for (const key of Object.keys(file.sitemap)) {
-            if (key === 'pathname')
+            if (key === 'pathname' || key === 'type')
                 continue
-            const value = file.sitemap[key];
-
-            if (typeof value === 'object' && value !== null) {
-                entry += `\t\t<${key}:${key}>\n`;
-
-                for (const nestedKey of Object.keys(value)) {
-                    const nestedValue = value[nestedKey];
-                    entry += `\t\t\t<${key}:${nestedKey}>${nestedValue}</${key}:${nestedKey}>\n`;
+            let value = file.sitemap[key];
+
+            if (typeof value === 'object' && value !== null && !(value instanceof Date)) {
+                if (!Array.isArray(value))
+                    value = [value]
+                for (let i = 0; i < value.length; i++) {
+                    entry += `\t\t<${key}:${key}>\n`;
+
+                    for (const nestedKey of Object.keys(value[i])) {
+                        const nestedValue = value[i][nestedKey];
+
+                        // Handle nested objects
+                        if (typeof nestedValue === 'object' && nestedValue !== null && !(nestedValue instanceof Date)) {
+                            entry += `\t\t\t<${key}:${nestedKey}>\n`;
+                            for (const subKey of Object.keys(nestedValue)) {
+                                const subValue = nestedValue[subKey];
+                                entry += `\t\t\t\t<${key}:${subKey}>${subValue}</${key}:${subKey}>\n`;
+                            }
+                            entry += `\t\t\t</${key}:${nestedKey}>\n`;
+                        } else {
+                            entry += `\t\t\t<${key}:${nestedKey}>${nestedValue}</${key}:${nestedKey}>\n`;
+                        }
+                    }
+
+                    entry += `\t\t</${key}:${key}>\n`;
                 }
-
-                entry += `\t\t</${key}:${key}>\n`;
             } else {
                 entry += `\t\t<${key}>${value}</${key}>\n`;
             }
@@ -153,7 +181,6 @@ class CoCreateSitemap {
         // Update loc using pathname
         file.sitemap.loc = `${file.pathname}`
 
-
         // Query the database for the correct sitemap based on the loc and type
         if (file.sitemap.pathname) {
             sitemap = await this.readSitemap(sitemap, host);
@@ -167,8 +194,8 @@ class CoCreateSitemap {
                 type = 'image';
             } else if (file['content-type'].startsWith('video/')) {
                 type = 'video';
-            } else if (file.sitemap.news) {
-                // type = 'news';
+            } else if (file.sitemap.type === 'news') {
+                type = 'news';
             }
 
             let name = `sitemap`
@@ -180,24 +207,37 @@ class CoCreateSitemap {
             if (index) {
                 sitemap.pathname = `/${name}${index}.xml`
                 sitemap = await this.readSitemap(sitemap, host);
+            } else {
+                index = 1
             }
 
             // Check if there's room in the last index sitemap
             if (!this.checkSitemap(sitemap.src)) {
                 if (sitemap.src)
                     index += 1
+                else
+                    sitemap.src = this.createSitemap(type);
+
                 sitemap.name = `${name}${index}.xml`
                 sitemap.pathname = `/${name}${index}.xml`
-                sitemap.src = this.createSitemap(type);
-
-                // Add the new sitemap entry
-                const indexEntry = `\n<sitemap>\n\t<loc>{{$host}}/${name}${index}.xml</loc>\n</sitemap>`;
-                mainSitemap.src = mainSitemap.src.replace('</sitemapindex>', `${indexEntry}\n</sitemapindex>`);
 
             }
 
         }
 
+        // Create the regex pattern to match the <sitemap> block containing the specific <loc> for the pathname
+        const regexPattern = `<sitemap>\\s*<loc>[^<]*${sitemap.pathname}[^<]*</loc>[\\s\\S]*?</sitemap>`;
+
+        // Execute the regex match against the sitemap index source
+        const match = mainSitemap.src.match(new RegExp(regexPattern));
+
+        // Check if a match is found
+        if (!match) {
+            //TODO: if sitemap found but not in index should we add to sitemap pathname to index or should we check the sitmap for the next index available see if room add or create new index. 
+            const indexEntry = `\t<sitemap>\n\t\t<loc>{{$host}}${sitemap.pathname}</loc>\n</sitemap>`;
+            mainSitemap.src = mainSitemap.src.replace('</sitemapindex>', `${indexEntry}\n</sitemapindex>`);
+        }
+
         return { mainSitemap, sitemap }
     }
 
@@ -223,12 +263,24 @@ class CoCreateSitemap {
     }
 
     createSitemap(type) {
-        if (type === 'main')
-            return `<?xml version="1.0" encoding="UTF-8"?>\n<sitemapindex xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">\n</sitemapindex>`;
-        else if (type === 'image' || type === 'video' || type === 'news')
-            return `<?xml version="1.0" encoding="UTF-8"?>\n<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9" xmlns:${type}="http://www.google.com/schemas/sitemap-${type}/1.1">\n</urlset>`;
-        else
-            return `<?xml version="1.0" encoding="UTF-8"?>\n<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">\n</urlset>`;
+        const xmlDeclaration = `<?xml version="1.0" encoding="UTF-8"?>\n`;
+        const sitemapNamespace = 'xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"';
+
+        if (type === 'main') {
+            return `${xmlDeclaration}<sitemapindex ${sitemapNamespace}>\n</sitemapindex>`;
+        } else {
+            const imageNamespace = 'xmlns:image="http://www.google.com/schemas/sitemap-image/1.1"';
+            const videoNamespace = 'xmlns:video="http://www.google.com/schemas/sitemap-video/1.1"';
+            const newsNamespace = 'xmlns:news="http://www.google.com/schemas/sitemap-news/0.9"';
+
+            if (type === 'image') {
+                return `${xmlDeclaration}<urlset ${sitemapNamespace} ${imageNamespace}>\n</urlset>`;
+            } else if (type === 'video') {
+                return `${xmlDeclaration}<urlset ${sitemapNamespace} ${videoNamespace}>\n</urlset>`;
+            } else { // For 'news' type or any other types
+                return `${xmlDeclaration}<urlset ${sitemapNamespace} ${newsNamespace} ${imageNamespace} ${videoNamespace}>\n</urlset>`;
+            }
+        }
     }
 
     async saveSitemap(file, host) {
@@ -259,7 +311,7 @@ class CoCreateSitemap {
             const regex = new RegExp(`\\/${filename}(\\d*)\\.xml<\\/loc>`, 'g');
             const matches = mainSitemap.src.match(regex);
 
-            return matches ? matches.length : 0;
+            return matches ? matches.length : null;
         } catch (err) {
             console.error(`Error determining next sitemap index for ${filename}:`, err);
             return null; // Or some default value or throw an error
@@ -293,6 +345,101 @@ class CoCreateSitemap {
         }
     }
 
+    parseHtml(file) {
+        const dom = parse(file.src);
+        const entries = dom.querySelectorAll('[sitemap="true"]');
+
+        let types = ['image', 'video', 'news']
+
+        const previousEntries = {}
+        for (let i = 0; i < types.length; i++) {
+            if (!file.sitemap[types[i]])
+                continue
+            if (Array.isArray(file.sitemap[types[i]]))
+                previousEntries[types[i]] = file.sitemap[types[i]]
+            else
+                previousEntries[types[i]] = [file.sitemap[types[i]]]
+
+            delete file.sitemap[types[i]]
+        }
+
+        for (let i = 0; i < entries.length; i++) {
+            let type = '', query = '';
+            let existingObject
+            let entryObject = {};
+
+            if (entries[i].tagName === 'IMG') {  // Corrected to 'IMG' for images
+                type = 'image';
+                query = 'loc'
+                entryObject.loc = entries[i].getAttribute('src');
+                entryObject.title = entries[i].getAttribute('sitemap-title') || entries[i].getAttribute('title') || entries[i].getAttribute('alt');
+                entryObject.caption = entries[i].getAttribute('sitemap-caption') || entries[i].getAttribute('alt') || entryObject.title;
+                entryObject.geo_location = entries[i].getAttribute('sitemap-geo-location');
+            } else if (entries[i].tagName === 'VIDEO') {
+                type = 'video';
+                query = 'content_loc'
+                entryObject.content_loc = entries[i].src;
+                entryObject.title = entries[i].getAttribute('sitemap-title') || entries[i].getAttribute('title');
+                entryObject.description = entries[i].getAttribute('description');  // 'description' if available
+                entryObject.thumbnail_loc = entries[i].getAttribute('sitemap-thumbnail') || entries[i].getAttribute('poster');
+                entryObject.duration = entries[i].getAttribute('sitemap-duration');
+            } else {
+                type = 'news';
+                file.sitemap.type = 'news';
+                query = 'title'
+                entryObject.title = entries[i].getAttribute('sitemap-title');
+                if (!entryObject.title) {
+                    const title = dom.querySelector('title');
+                    entryObject.title = title ? title.text : '';
+                }
+
+                entryObject.publication = {
+                    name: entries[i].getAttribute('sitemap-publication-name'),  // Use proper attribute
+                    language: entries[i].getAttribute('sitemap-publication-language')  // Use proper attribute
+                };
+
+                if (!entryObject.publication.language) {
+                    // Fallback to HTML lang attribute
+                    const htmlElement = dom.querySelector('html');
+                    entryObject.publication.language = htmlElement ? htmlElement.getAttribute('lang') : null;
+                }
+
+                entryObject.publication_date = entries[i].getAttribute('sitemap-publication-date') || file.modified.on;
+
+                entryObject.keywords = entries[i].getAttribute('sitemap-keywords');
+                if (!entryObject.keywords) {
+                    const keywords = dom.querySelector('meta[name="keywords"]');
+                    entryObject.keywords = keywords ? keywords.getAttribute('content') : '';
+                }
+
+                entryObject.genres = entries[i].getAttribute('sitemap-genres');
+            }
+
+            if (previousEntries[type]) {
+                existingObject = previousEntries[type].find(item => item[query] === entryObject[query]);
+                entryObject = { ...existingObject, ...entryObject }
+            }
+
+            Object.keys(entryObject).forEach(key => {
+                if (!entryObject[key])
+                    delete entryObject[key]
+            });
+
+            if (!file.sitemap[type])
+                file.sitemap[type] = []
+
+            file.sitemap[type].push(entryObject)
+
+        }
+
+        if (file.sitemap.type !== 'news') {
+            const priorityMeta = dom.querySelector('meta[name="sitemap-priority"]');
+            const changefreqMeta = dom.querySelector('meta[name="sitemap-changefreq"]');
+            file.sitemap.priority = priorityMeta ? priorityMeta.getAttribute('content') : file.sitemap.priority; // Default priority if not specified
+            file.sitemap.changefreq = changefreqMeta ? changefreqMeta.getAttribute('content') : file.sitemap.changefreq; // Default changefreq if not specified
+        }
+
+    }
 }
 
 module.exports = CoCreateSitemap;