hdoc-tools 0.8.35 → 0.8.37

package/hdoc-build-pdf.js

@@ -64,17 +64,21 @@
         }).get();
         imgs.push(...srcs);
         for (let i = 0; i < imgs.length; i++) {
-            if (hdoc.valid_url(imgs[i]).valid) {
+            if (hdoc.valid_url(imgs[i])) {
                 // External Link
-                const file_response = request('GET', imgs[i]);
-                if (file_response.statusCode === 200) {
-                    const image_buffer = file_response.getBody();
-                    const mime_type = mime.lookup(imgs[i]);
-                    let image_b64 = image_buffer.toString("base64");
-                    image_b64 = `data:${mime_type};base64,${image_b64}`;
-                    html_source = html_source.replace(imgs[i], image_b64);
-                } else {
-                    throw `Unexpected Status ${file_response.statusCode}`;
+                try {
+                    const file_response = request('GET', imgs[i]);
+                    if (file_response.statusCode === 200) {
+                        const image_buffer = file_response.getBody();
+                        const mime_type = mime.lookup(imgs[i]);
+                        let image_b64 = image_buffer.toString("base64");
+                        image_b64 = `data:${mime_type};base64,${image_b64}`;
+                        html_source = html_source.replace(imgs[i], image_b64);
+                    } else {
+                        throw `Unexpected Status ${file_response.statusCode}`;
+                    }
+                } catch (err) {
+                    console.log(`Error downloading external source [${imgs[i]}] - ${err}`);
                 }
             } else {
                 // Internal link
@@ -86,7 +90,7 @@
                     image_b64 = `data:${mime_type};base64,${image_b64}`;
                     html_source = html_source.replace(imgs[i], image_b64);
                 } catch (err) {
-                    console.log('Error reading image from HTML source [', image_path, '] - ', err);
+                    console.log('Error reading image from HTML source [', image_path, '] -', err);
                     return null;
                 }
             }

package/hdoc-build.js

@@ -652,15 +652,12 @@
                                         });
                                     }
                                 }
-
                             }
                         }
                     }
                 }
             }
-
         }
-        //console.log(JSON.stringify(bc, null, 2));
     };
 
     exports.run = async function (source_path, verbose_output, github_api_token) {
@@ -765,7 +762,7 @@
         console.log(`       Static HTML Files Found: ${static_html_files.length}\n`);
 
         // Validate content
-        const validation_success = validate.run(work_path, doc_id, verbose);
+        const validation_success = await validate.run(work_path, doc_id, verbose, hdocbook_config, hdocbook_project);
         if (!validation_success) {
             process.exit(1);
         }

package/hdoc-module.js

@@ -2,8 +2,8 @@
     'use strict';
 
     const   cheerio = require('cheerio'),
-            request = require('sync-request'),
             html2text = require('html-to-text'),
+            request = require('sync-request'),
             wordsCount = require('words-count').default;
 
     let includesCache = {};
@@ -54,18 +54,17 @@
         }
     };
 
-    exports.valid_url = function (s) {
-        let response = {
-            valid: false,
-            urlProps: {}
-        };
-        try {
-            response.urlProps = new URL(s);
-            response.valid = true;
-        } catch (err) {
-            response.valid = false;
-        }
-        return response;
+    exports.valid_url = function (url) {
+
+        const stringIsAValidUrl = (s) => {
+            try {
+              new URL(s);
+              return true;
+            } catch (err) {
+              return false;
+            }
+          };
+        return stringIsAValidUrl(url);
     };
 
     exports.expand_variables = function (text, docId = '') {

package/hdoc-validate.js

@@ -1,34 +1,79 @@
 (function () {
     'use strict';
 
-    const cheerio = require('cheerio'),
+    const axios = require('axios'),
+        cheerio = require('cheerio'),
         dree = require('dree'),
         fs = require('fs'),
+        https = require('https'),
         path = require('path'),
-        hdoc = require(path.join(__dirname, 'hdoc-module.js')),
-        URL = require("url").URL;
+        hdoc = require(path.join(__dirname, 'hdoc-module.js'));
+
+        const agent = new https.Agent({  
+            rejectUnauthorized: false
+        });
 
     let errors = {},
         messages = {},
         warnings = {},
         errorcount = 0,
-        filecount = 0,
-        htmlFiles = [];
+        html_files = [],
+        exclude_links = {};
 
 
-    const checkLinks = function (source_path, htmlFile, links) {
+    const checkLinks = async function (source_path, htmlFile, links, hdocbook_config) {
         for (let i = 0; i < links.length; i++) {
 
             // Validate that link is a valid URL first
-            if (!hdoc.valid_url(links[i]).valid) {
-
+            if (!hdoc.valid_url(links[i])) {
                 // Could be a relative path, check
-                const fileExists = doesFileExist(source_path, htmlFile, links[i]);
-                if (!fileExists) {
-                    errorcount++;
+                isRelativePath(source_path, htmlFile, links[i]);
+            } else {
+                messages[htmlFile.relativePath].push(`Link is a properly formatted external URL: ${links[i]}`);
+
+                // Skip if it's the auto-generated edit url, as these could be part of a private repo which would return a 404
+                if (links[i] === hdoc.get_github_api_path(hdocbook_config.publicSource, htmlFile.relativePath).edit_path.replace(path.extname(htmlFile.relativePath), '.md')) {
+                    continue;
+                }
+
+                // Skip if the link is excluded in the project config
+                if (exclude_links[links[i]]) {
+                    continue;
                 }
+
+                // Skip mailto links
+                if (links[i].startsWith('mailto:')) {
+                    continue;
+                }
+
+                try {
+                    await axios.get(links[i], { httpsAgent: agent });
+                    messages[htmlFile.relativePath].push(`Link is a valid external URL: ${links[i]}`);
+                } catch (e) {
+                    // Handle errors
+                    errors[htmlFile.relativePath].push(`Link is not responding: ${links[i]} - [${e.message}]`);
+                }
+            }
+        }
+    };
+
+    const checkImages = async function (source_path, htmlFile, links) {
+        for (let i = 0; i < links.length; i++) {
+
+            // Validate that image is a valid URL first
+            if (!hdoc.valid_url(links[i])) {
+                // Could be a relative path, check image exists
+                doesFileExist(source_path, htmlFile, links[i]);
             } else {
-                messages[htmlFile.relativePath].push(`Link is valid External URL: ${links[i]}`);
+                messages[htmlFile.relativePath].push(`Image link is a properly formatted external URL: ${links[i]}`);
+                // Do a Get to the URL to see if it exists
+                try {
+                    const res = await axios.get(links[i]);
+                    messages[htmlFile.relativePath].push(`Image link is a valid external URL: ${links[i]}`);
+                } catch (e) {
+                    // Handle errors
+                    errors[htmlFile.relativePath].push(`Unexpected Error from external image link: ${links[i]} - ${e.message}`);
+                }
             }
         }
     };
@@ -47,16 +92,57 @@
 
     // File callbacks for html file scan
     const fileCallback = function (element) {
-        filecount++;
-        htmlFiles.push(element);
+        html_files.push(element);
     };
 
-    const doesFileExist = function (source_path, html_path, relative_path) {
-        // Remove explicit anchor links
-        let file_path = path.join(source_path, relative_path.split('#')[0]);
+    const isRelativePath = function (source_path, html_path, relative_path) {
+        const rel_path_ext = path.extname(relative_path);
+        let response = {
+            is_rel_path: false,
+            has_md_extension: rel_path_ext === '.md'
+        };
+
+        const supported_relpaths = [
+            path.sep + 'index.htm',
+            path.sep + 'index.html',
+            '.htm',
+            '.html',
+            '.md'
+        ];
 
-        // Replace /_books/ with /
-        file_path = file_path.replace(path.sep + '_books' + path.sep, path.sep);
+        // Remove explicit anchor links and _books prefix
+        relative_path = relative_path.split('#')[0].replace('_books/', '');
+
+        // Make full file path
+        const file_path = path.join(source_path, relative_path);
+
+        // Does path exist?
+        if (fs.existsSync(file_path)) {
+            response.is_rel_path = true;
+        } else {
+            // Path 
+            for (let i = 0; i < supported_relpaths.length; i++) {
+                if (fs.existsSync(`${file_path}${supported_relpaths[i]}`)) {
+                    response.is_rel_path = true;
+                    break;
+                }
+            }
+        }
+        if (response.has_md_extension) {
+            errors[html_path.relativePath].push(`Relative link contains MD extension, but should not: ${relative_path}`);
+        } else {
+            if (response.is_rel_path) {
+                messages[html_path.relativePath].push(`Relative path exists: ${relative_path}`);
+            } else {
+                errors[html_path.relativePath].push(`Link path does not exist: ${relative_path}`);
+            }
+        }
+    }
+
+    const doesFileExist = function (source_path, html_path, relative_path) {
+        // Remove explicit anchor links and _books prefix
+        relative_path = relative_path.split('#')[0].replace('_books/', '');
+        const file_path = path.join(source_path, relative_path);
         if (!fs.existsSync(file_path) && !fs.existsSync(file_path + path.sep + 'index.htm') && !fs.existsSync(file_path + 'index.html') && !fs.existsSync(file_path + '.htm') && !fs.existsSync(file_path + '.html')) {
             errors[html_path.relativePath].push(`Book resource does not exist: ${relative_path}`);
             return false;
@@ -70,7 +156,10 @@
     const getLinks = function (file) {
         messages[file.relativePath].push('Parsing HTML file');
         const htmlBody = fs.readFileSync(file.path, 'utf8');
-        let links = [];
+        let links = {
+            href: [],
+            img: []
+        };
         const $ = cheerio.load(htmlBody);
         const hrefs = $('a').map(function (i) {
             return $(this).attr('href');
@@ -78,35 +167,46 @@
         const srcs = $('img').map(function (i) {
             return $(this).attr('src');
         }).get();
-        links.push(...hrefs);
-        links.push(...srcs);
+        links.href.push(...hrefs);
+        links.img.push(...srcs);
         return links;
     };
 
-    exports.run = function (source_path, doc_id, verbose) {
+    exports.run = async function (source_path, doc_id, verbose, hdocbook_config, hdocbook_project) {
         // Get a list of HTML files in source_path
         dree.scan(source_path, dreeOptions, fileCallback);
 
+        if (hdocbook_project.validation && hdocbook_project.validation.exclude_links && hdocbook_project.validation.exclude_links instanceof Array) {
+            hdocbook_project.validation.exclude_links.forEach(function(excl_link) {
+                exclude_links[excl_link] = true;
+            });
+        }
+
         console.log(`Performing Validation and Building SEO Link List...`);
 
         let listContent = '';
-        for (let i = 0; i < htmlFiles.length; i++) {
+        for (let i = 0; i < html_files.length; i++) {
 
             // Initiate maps for errors and verbose messages for HTML file
-            errors[htmlFiles[i].relativePath] = [];
-            messages[htmlFiles[i].relativePath] = [];
-            warnings[htmlFiles[i].relativePath] = [];
+            errors[html_files[i].relativePath] = [];
+            messages[html_files[i].relativePath] = [];
+            warnings[html_files[i].relativePath] = [];
 
-            const links = getLinks(htmlFiles[i]);
-            if (links.length === 0) {
-                messages[htmlFiles[i].relativePath].push('No links found in file');
+            const links = getLinks(html_files[i]);
+            if (links.href.length === 0) {
+                messages[html_files[i].relativePath].push('No links found in file');
+            } else {
+                await checkLinks(source_path, html_files[i], links.href, hdocbook_config);
+            }
+            if (links.img.length === 0) {
+                messages[html_files[i].relativePath].push('No images found in file');
             } else {
-                checkLinks(source_path, htmlFiles[i], links);
+                await checkImages(source_path, html_files[i], links.img);
             }
 
             // Build list content for Google 
-            listContent += `/${htmlFiles[i].relativePath.replace(path.extname(htmlFiles[i].relativePath), '')}`;
-            if (i < htmlFiles.length - 1) {
+            listContent += `/${html_files[i].relativePath.replace(path.extname(html_files[i].relativePath), '')}`;
+            if (i < html_files.length - 1) {
                 listContent += '\r\n';
             }
         }
@@ -118,24 +218,6 @@
         } catch (err) {
             console.error(err);
         }
-        if (errorcount === 0) {
-            console.log('\r\n---------------');
-            console.log('   No Errors   ');
-            console.log('---------------\r\n');
-        } else {
-            console.log('\r\n------------');
-            console.log('   Errors   ');
-            console.log('------------');
-
-            for (const key in errors) {
-                if (errors.hasOwnProperty(key) && errors[key].length > 0) {
-                    console.log(`\r\n${errors[key].length} error(s) in ${key}`);
-                    for (let i = 0; i < errors[key].length; i++) {
-                        console.log(` - ${errors[key][i]}`);
-                    }
-                }
-            }
-        }
 
         if (verbose) {
             console.log('\r\n-------------');
@@ -150,7 +232,26 @@
                 }
             }
         }
-        console.log(`\r\nValidation Errors Found: ${errorcount}\r\n`);
-        return errorcount === 0 ? true : false;
+
+        console.log('\r\n-----------------------');
+        console.log('   Validation Output   ');
+        console.log('-----------------------');
+
+        for (const key in errors) {
+            if (errors.hasOwnProperty(key) && errors[key].length > 0) {
+                console.log(`\r\n${errors[key].length} error(s) in ${key}`);
+                for (let i = 0; i < errors[key].length; i++) {
+                    console.log(` - ${errors[key][i]}`);
+                    errorcount++;
+                }
+            }
+        }
+
+        if (errorcount > 0) {
+            console.log(`\r\n${errorcount} Validation Errors Found\r\n`);
+            return false;
+        }
+        console.log(`\r\nNo Validation Errors Found!\r\n`);
+        return true;
     };
 })();

package/hdoc.js

@@ -1,5 +1,4 @@
-#!/usr/bin/env node
-
+#!/usr/bin/env node
 (async function () {
    'use strict';
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "hdoc-tools",
-  "version": "0.8.35",
+  "version": "0.8.37",
   "description": "Hornbill HDocBook Development Support Tool",
   "main": "hdoc.js",
   "bin": {
@@ -30,6 +30,7 @@
   "author": "Hornbill Technologies Ltd",
   "license": "ISC",
   "dependencies": {
+    "axios": "^1.3.2",
     "better-sqlite3": "^8.0.1",
     "body-parser": "^1.20.1",
     "cheerio": "^1.0.0-rc.12",
@@ -40,7 +41,6 @@
     "highlight.js": "^11.6.0",
     "html-to-text": "^8.2.1",
     "js-yaml": "^4.1.0",
-    "jspdf": "^2.5.1",
     "markdown-it": "^13.0.1",
     "markdown-it-container": "^3.0.0",
     "markdown-it-front-matter": "^0.2.3",