@cocreate/sitemap 1.0.0 → 1.2.0

package/CHANGELOG.md

@@ -1,3 +1,31 @@
+# [1.2.0](https://github.com/CoCreate-app/CoCreate-sitemap/compare/v1.1.0...v1.2.0) (2024-08-30)
+
+
+### Bug Fixes
+
+* add sitemap to sitemapindex if not already in index ([0ff6911](https://github.com/CoCreate-app/CoCreate-sitemap/commit/0ff6911fe81eeb05295639fb984401d3e76b90ad))
+* noindex metatag return and handling sitemap generation ([058116a](https://github.com/CoCreate-app/CoCreate-sitemap/commit/058116a34c44100bfa4ea1b20afe13fd489bc642))
+
+
+### Features
+
+* added node-html-parser ([397c590](https://github.com/CoCreate-app/CoCreate-sitemap/commit/397c5908af1149d2d60d7c431a8a0f1bd60789f4))
+* parseHtml function to read and generate nested sitemaps using sitemap-* attributes and metatags ([64edbdb](https://github.com/CoCreate-app/CoCreate-sitemap/commit/64edbdbdcee6f64bc5080997557f9691f072381c))
+
+# [1.1.0](https://github.com/CoCreate-app/CoCreate-sitemap/compare/v1.0.0...v1.1.0) (2024-08-24)
+
+
+### Bug Fixes
+
+* apply host for crud ([9a9fbf0](https://github.com/CoCreate-app/CoCreate-sitemap/commit/9a9fbf0c636d6aa50a607e9fdd2cf96c29f0e1bc))
+* check logic to auto add html files to sitemap ([6ca692b](https://github.com/CoCreate-app/CoCreate-sitemap/commit/6ca692b486ad2f26c7140349756ebc7454ce2e00))
+* handle mainSitmap and sitmap as a file object ([99b97c3](https://github.com/CoCreate-app/CoCreate-sitemap/commit/99b97c3aa3198ea090a2efe8d9d2af5875af73ed))
+
+
+### Features
+
+* create multiple sitemaps for the various supported types ([98af2dc](https://github.com/CoCreate-app/CoCreate-sitemap/commit/98af2dc972ae15a34caaf709358d543f67828e95))
+
 # 1.0.0 (2024-06-27)
 
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@cocreate/sitemap",
-  "version": "1.0.0",
+  "version": "1.2.0",
   "description": "A simple sitemap component in vanilla javascript. Easily configured using HTML5 data-attributes and/or JavaScript API.",
   "keywords": [
     "sitemap",
@@ -43,5 +43,8 @@
     "type": "GitHub Sponsors ❤",
     "url": "https://github.com/sponsors/CoCreate-app"
   },
-  "main": "./src/index.js"
+  "main": "./src/index.js",
+  "dependencies": {
+    "node-html-parser": "^6.1.13"
+  }
 }

package/src/index.js

@@ -20,53 +20,410 @@
 // you must obtain a commercial license from CoCreate LLC.
 // For details, visit <https://cocreate.app/licenses/> or contact us at sales@cocreate.app.
 
-const fs = require('fs');
+const { parse } = require("node-html-parser");
 
 class CoCreateSitemap {
-    constructor(render) {
-        this.render = render;
+    constructor(crud) {
+        this.crud = crud;
     }
 
-    async updateUrlInSitemap(urlToFind) {
-        try {
-            // Path to your sitemap file
-            const sitemapPath = '/path/to/your/sitemap.xml';
+    async check(file, host) {
+        // Ensure the file is HTML and does not have a sitemap object yet
+        if (!file.sitemap && file['content-type'] !== 'text/html')
+            return;
 
-            // Read and parse the sitemap XML
-            let sitemapXml = fs.readFileSync(sitemapPath, 'utf8');
+        // Ensure the file is public
+        if (!file.public || file.public === "false")
+            return;
 
-            // Regex pattern to find the entire <url>...</url> block containing the URL
-            const regexPattern = `<url>\\s*<loc>${urlToFind}</loc>[\\s\\S]*?</url>`;
+        // Check if the file is HTML and contains a noindex meta tag
+        if (file['content-type'] === 'text/html'
+            && /<meta\s+name=["']robots["']\s+content=["'][^"']*noindex[^"']*["']/i.test(file.src))
+            return;
 
-            // Perform regex search
-            const match = sitemapXml.match(regexPattern);
+        // Compare the lastmod date in the sitemap with the modified.on date
+        if (file.sitemap && file.sitemap.lastmod && file.modified.on) {
+            if (new Date(file.sitemap.lastmod) >= new Date(file.modified.on))
+                return;
+        }
 
-            if (match) {
-                const position = match.index; // Start position of the <url> block
-                const endPosition = match.index + match[0].length; // End position of the <url> block
-                console.log(`URL ${urlToFind} found in sitemap at position ${position}-${endPosition}.`);
+        // Logic to update the sitemap
+        this.updateSitemap(file, host);
+    }
 
-                // Replace the matched <url> block with a modified version (example)
-                const modifiedUrlBlock = `<url>
-          <loc>${urlToFind}</loc>
-          <lastmod>${new Date().toISOString()}</lastmod>
-          <priority>1.0</priority>
-        </url>`;
+    async updateSitemap(file, host) {
+        try {
+            // TODO: need to get info such as host
+            const entry = this.createEntry(file);
 
-                // Replace the original <url> block with the modified one
-                sitemapXml = sitemapXml.slice(0, position) + modifiedUrlBlock + sitemapXml.slice(endPosition);
+            let { mainSitemap, sitemap } = await this.getSitemap(file, host);
 
-                // Write back the modified sitemap XML to the file (optional)
-                fs.writeFileSync(sitemapPath, sitemapXml);
+            if (file.pathname) {
+                // Perform regex search starting at the pathname
+                const regexPattern = `<url>\\s*<loc>.*?${file.pathname.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}.*?</loc>[\\s\\S]*?</url>`;
+                const match = sitemap.src.match(new RegExp(regexPattern));
 
-                console.log('Sitemap updated successfully.');
+                if (match) {
+                    const position = match.index; // Start position of the <url> block
+                    const endPosition = match.index + match[0].length; // End position of the <url> block
+
+                    // Replace the original <url> block with the modified one
+                    sitemap.src = sitemap.src.slice(0, position) + entry + sitemap.src.slice(endPosition);
+                } else {
+                    sitemap.src = sitemap.src.replace('</urlset>', `${entry}</urlset>`);
+                }
             } else {
-                console.log(`URL ${urlToFind} not found in sitemap.`);
+                file.sitemap.pathname = sitemap.pathname
+                sitemap.src = sitemap.src.replace('</urlset>', `${entry}</urlset>`);
             }
+
+            this.saveSitemap(mainSitemap, host);
+            this.saveSitemap(sitemap, host);
+            this.saveSitemap(file, host);
+
+            // console.log('Sitemap updated successfully.');
         } catch (err) {
             console.error('Error updating sitemap:', err);
         }
     }
+
+    createEntry(file) {
+        file.sitemap.loc = file.pathname;
+        file.sitemap.lastmod = file.modified.on;
+
+        if (file['content-type'] === 'text/html') {
+            parseHtml(file)
+
+            if (file.sitemap.type !== 'news') {
+                if (file.sitemap.changefreq)
+                    file.sitemap.changefreq = 'monthly';
+
+                if (!file.sitemap.priority) {
+                    const depth = (file.pathname.match(/\//g) || []).length;
+                    file.sitemap.priority = Math.max(0.1, 1.0 - (depth - 1) * 0.1).toFixed(1);
+                }
+            }
+        }
+
+        let entry = `\t<url>\n`;
+
+        for (const key of Object.keys(file.sitemap)) {
+            if (key === 'pathname' || key === 'type')
+                continue
+            let value = file.sitemap[key];
+
+            if (typeof value === 'object' && value !== null && !(value instanceof Date)) {
+                if (!Array.isArray(value))
+                    value = [value]
+                for (let i = 0; i < value.length; i++) {
+                    entry += `\t\t<${key}:${key}>\n`;
+
+                    for (const nestedKey of Object.keys(value[i])) {
+                        const nestedValue = value[nestedKey];
+                        entry += `\t\t\t<${key}:${nestedKey}>${nestedValue}</${key}:${nestedKey}>\n`;
+                    }
+
+                    entry += `\t\t</${key}:${key}>\n`;
+                }
+            } else {
+                entry += `\t\t<${key}>${value}</${key}>\n`;
+            }
+        }
+
+        entry += `\t</url>\n`;
+
+        return entry;
+    }
+
+    async getSitemap(file, host) {
+        let mainSitemap = {
+            host: file.host,
+            name: 'sitemap.xml',
+            path: '/',
+            pathname: '/sitemap.xml',
+            directory: '/',
+            'content-type': 'application/xml',
+            public: true,
+            organization_id: file.organization_id
+        }
+
+        mainSitemap = await this.readSitemap(mainSitemap, host);
+        if (!mainSitemap.src)
+            mainSitemap.src = this.createSitemap('main')
+
+        let sitemap = {
+            host: file.host,
+            path: '/',
+            pathname: file.sitemap.pathname,
+            directory: '/',
+            'content-type': 'application/xml',
+            public: true,
+            organization_id: file.organization_id
+        }
+
+        // Update loc using pathname
+        file.sitemap.loc = `${file.pathname}`
+
+
+        // Query the database for the correct sitemap based on the loc and type
+        if (file.sitemap.pathname) {
+            sitemap = await this.readSitemap(sitemap, host);
+        }
+
+        if (!sitemap.src) {
+            let type = 'sitemap';
+
+            // Identify content type to determine sitemap type
+            if (file['content-type'].startsWith('image/')) {
+                type = 'image';
+            } else if (file['content-type'].startsWith('video/')) {
+                type = 'video';
+            } else if (file.sitemap.type === 'news') {
+                type = 'news';
+            }
+
+            let name = `sitemap`
+            if (type === 'image' || type === 'video' || type === 'news')
+                name = `sitemap-${type}`
+
+            // If no existing sitemap found check last index sitemap
+            let index = await this.getLastSitemapIndex(mainSitemap, name);
+            if (index) {
+                sitemap.pathname = `/${name}${index}.xml`
+                sitemap = await this.readSitemap(sitemap, host);
+            } else {
+                index = 1
+            }
+
+            // Check if there's room in the last index sitemap
+            if (!this.checkSitemap(sitemap.src)) {
+                if (sitemap.src)
+                    index += 1
+                else
+                    sitemap.src = this.createSitemap(type);
+
+                sitemap.name = `${name}${index}.xml`
+                sitemap.pathname = `/${name}${index}.xml`
+
+            }
+
+        }
+
+        // Create the regex pattern to match the <sitemap> block containing the specific <loc> for the pathname
+        const regexPattern = `<sitemap>\\s*<loc>[^<]*${sitemap.pathname}[^<]*</loc>[\\s\\S]*?</sitemap>`;
+
+        // Execute the regex match against the sitemap index source
+        const match = mainSitemap.src.match(new RegExp(regexPattern));
+
+        // Check if a match is found
+        if (!match) {
+            //TODO: if sitemap found but not in index should we add to sitemap pathname to index or should we check the sitmap for the next index available see if room add or create new index. 
+            const indexEntry = `\t<sitemap>\n\t\t<loc>{{$host}}${sitemap.pathname}</loc>\n</sitemap>`;
+            mainSitemap.src = mainSitemap.src.replace('</sitemapindex>', `${indexEntry}\n</sitemapindex>`);
+        }
+
+        return { mainSitemap, sitemap }
+    }
+
+    async readSitemap(file, host) {
+        let data = {
+            method: 'object.read',
+            host: host,
+            array: 'files',
+            $filter: {
+                query: {
+                    host: { $in: [host, '*'] },
+                    pathname: file.pathname
+                },
+                limit: 1
+            },
+            organization_id: file.organization_id
+        }
+        data = await this.crud.send(data)
+        if (data.object && data.object.length)
+            return data.object[0]
+        else
+            return file
+    }
+
+    createSitemap(type) {
+        const xmlDeclaration = `<?xml version="1.0" encoding="UTF-8"?>\n`;
+        const sitemapNamespace = 'xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"';
+
+        if (type === 'main') {
+            return `${xmlDeclaration}<sitemapindex ${sitemapNamespace}>\n</sitemapindex>`;
+        } else {
+            const imageNamespace = 'xmlns:image="http://www.google.com/schemas/sitemap-image/1.1"';
+            const videoNamespace = 'xmlns:video="http://www.google.com/schemas/sitemap-video/1.1"';
+            const newsNamespace = 'xmlns:news="http://www.google.com/schemas/sitemap-news/0.9"';
+
+            if (type === 'image') {
+                return `${xmlDeclaration}<urlset ${sitemapNamespace} ${imageNamespace}>\n</urlset>`;
+            } else if (type === 'video') {
+                return `${xmlDeclaration}<urlset ${sitemapNamespace} ${videoNamespace}>\n</urlset>`;
+            } else { // For 'news' type or any other types
+                return `${xmlDeclaration}<urlset ${sitemapNamespace} ${newsNamespace} ${imageNamespace} ${videoNamespace}>\n</urlset>`;
+            }
+        }
+    }
+
+    async saveSitemap(file, host) {
+        let data = {
+            method: 'object.update',
+            host: host,
+            array: 'files',
+            object: file,
+            upsert: true,
+            organization_id: file.organization_id
+        }
+        if (!file._id)
+            data.$filter = {
+                query: {
+                    host: { $in: [host, '*'] },
+                    pathname: file.pathname
+                },
+                limit: 1
+            }
+
+        data = await this.crud.send(data)
+
+    }
+
+    async getLastSitemapIndex(mainSitemap, filename) {
+        try {
+            // Use regex to match all sitemap entries for the given type
+            const regex = new RegExp(`\\/${filename}(\\d*)\\.xml<\\/loc>`, 'g');
+            const matches = mainSitemap.src.match(regex);
+
+            return matches ? matches.length : null;
+        } catch (err) {
+            console.error(`Error determining next sitemap index for ${filename}:`, err);
+            return null; // Or some default value or throw an error
+        }
+    }
+
+    checkSitemap(sitemap) {
+        try {
+            if (!sitemap)
+                return false
+
+            // Count the number of <url> entries
+            const urlCount = (sitemap.match(/<url>/g) || []).length;
+            if (urlCount >= 50000)
+                return false;
+
+            // Get the size of the sitemap string in bytes
+            const fileSizeInBytes = Buffer.byteLength(sitemap, 'utf8');
+            const fileSizeInMB = fileSizeInBytes / (1024 * 1024);
+
+            // console.log(`Sitemap has ${urlCount} entries and is ${fileSizeInMB.toFixed(2)} MB.`);
+
+            // Check if the file size exceeds either the 50MB limit or the MongoDB 16MB limit
+            if (fileSizeInMB >= 50 || fileSizeInMB >= 15)
+                return false;
+
+            return true;
+        } catch (err) {
+            console.error('Error checking sitemap file:', err);
+            return false;
+        }
+    }
+
+    parseHtml(file) {
+        const dom = parse(html);
+        const entries = dom.querySelectorAll('[sitemap="true"]');
+
+        let types = ['image', 'video', 'news']
+
+        const previousEntries = {}
+        for (let i = 0; i < types.length; i++) {
+            if (!file.sitemap[types[i]])
+                continue
+            if (Array.isArray(file.sitemap[types[i]]))
+                previousEntries[types[i]] = file.sitemap[types[i]]
+            else
+                previousEntries[types[i]] = [file.sitemap[types[i]]]
+
+            delete file.sitemap[types[i]]
+        }
+
+        for (let i = 0; i < entries.length; i++) {
+            let type = '', query = '';
+            let existingObject
+            let entryObject = {};
+
+            if (entries[i].tagName === 'IMG') {  // Corrected to 'IMG' for images
+                type = 'image';
+                query = 'loc'
+                entryObject.loc = entries[i].src;
+                entryObject.title = entries[i].getAttribute('sitemap-title') || entries[i].getAttribute('title') || entries[i].getAttribute('alt');
+                entryObject.caption = entries[i].getAttribute('sitemap-caption') || entries[i].getAttribute('alt') || entryObject.title;
+                entryObject.geo_location = entries[i].getAttribute('sitemap-geo-location');
+            } else if (entries[i].tagName === 'VIDEO') {
+                type = 'video';
+                query = 'content_loc'
+                entryObject.content_loc = entries[i].src;
+                entryObject.title = entries[i].getAttribute('sitemap-title') || entries[i].getAttribute('title');
+                entryObject.description = entries[i].getAttribute('description');  // 'description' if available
+                entryObject.thumbnail_loc = entries[i].getAttribute('sitemap-thumbnail') || entries[i].getAttribute('poster');
+                entryObject.duration = entries[i].getAttribute('sitemap-duration');
+            } else {
+                type = 'news';
+                file.sitemap.type = 'news';
+                query = 'title'
+                entryObject.title = entries[i].getAttribute('sitemap-title');
+                if (!entryObject.title) {
+                    const title = dom.querySelector('title');
+                    entryObject.title = title ? title.text : '';
+                }
+
+                entryObject.publication = {
+                    name: entries[i].getAttribute('sitemap-publication-name'),  // Use proper attribute
+                    language: entries[i].getAttribute('sitemap-publication-language')  // Use proper attribute
+                };
+
+                if (!entryObject.publication.language) {
+                    // Fallback to HTML lang attribute
+                    const htmlElement = dom.querySelector('html');
+                    entryObject.publication.language = htmlElement ? htmlElement.getAttribute('lang') : null;
+                }
+
+                entryObject.publication_date = entries[i].getAttribute('sitemap-publication-date') || file.modified.on;
+
+                entryObject.keywords = entries[i].getAttribute('sitemap-keywords');
+                if (!entryObject.keywords) {
+                    const keywords = dom.querySelector('meta[name="keywords"]');
+                    entryObject.keywords = keywords ? keywords.getAttribute('content') : '';
+                }
+
+                entryObject.genres = entries[i].getAttribute('sitemap-genres');
+            }
+
+            if (previousEntries[type]) {
+                existingObject = previousEntries[type].find(item => item[query] === entryObject[query]);
+                entryObject = { ...existingObject, ...entryObject }
+            }
+
+            Object.keys(entryObject).forEach(key => {
+                if (!entryObject[key])
+                    delete entryObject[key]
+            });
+
+            if (!file.sitemap[type])
+                file.sitemap[type] = []
+
+            file.sitemap[type].push(entryObject)
+
+        }
+
+        if (file.sitemap.type !== 'news') {
+            const priorityMeta = dom.querySelector('meta[name="sitemap-priority"]');
+            const changefreqMeta = dom.querySelector('meta[name="sitemap-changefreq"]');
+            file.sitemap.priority = priorityMeta ? priorityMeta.getAttribute('content') : file.sitemap.priority; // Default priority if not specified
+            file.sitemap.changefreq = changefreqMeta ? changefreqMeta.getAttribute('content') : file.sitemap.changefreq; // Default changefreq if not specified
+        }
+
+    }
 }
 
 module.exports = CoCreateSitemap;