hdoc-tools 0.9.7 → 0.9.9

package/hdoc-build-db.js

@@ -14,7 +14,8 @@
             'doc_title',
             'doc_content',
             'doc_preview UNINDEXED',
-            'doc_family_id'
+            'doc_family_id',
+            'doc_md5_hash UNINDEXED'
         ],
         hdoc_meta: [
             'resource_url',
@@ -89,7 +90,8 @@
                 index_records[i].index_html.fm_props.title,
                 index_records[i].index_html.text,
                 index_records[i].index_html.preview,
-                book_config.productFamily
+                book_config.productFamily,
+                index_records[i].md5
             ];
             const index_response = await  hdoc_index.insert_record(db, 'hdoc_index', db_schema.hdoc_index, index_vals);
             if (!index_response.success) {

package/hdoc-build.js

@@ -45,6 +45,7 @@
         includes_failed = 0,
         index_records = [],
         md_files = [],
+        md_files_delete = [],
         static_html_files = [],
         work_path_content = '',
         verbose = false;
@@ -246,7 +247,8 @@
                 index_html: hdoc_index.transform_html_for_index(html_txt),
                 metadata: metadata,
                 contributors: contribs,
-                pdf_size: pdf_size
+                pdf_size: pdf_size,
+                md5: file_path.hash
             });
             // Save HTML into HTML file
             try {
@@ -259,6 +261,7 @@
 
     const transform_markdown_and_save_html = async function (file_path) {
         conversion_attempted++;
+
         if (fs.existsSync(file_path.path)) {
             // Load markdown file
             let md_txt = hdoc.expand_variables(fs.readFileSync(file_path.path, 'utf8'));
@@ -465,15 +468,13 @@
                 index_html: index_details,
                 metadata: metadata,
                 contributors: contribs,
-                pdf_size: pdf_size
+                pdf_size: pdf_size,
+                md5: file_path.hash
             });
 
-            // Delete MD file from _work path
-            try {
-                fs.unlinkSync(file_path.path);
-            } catch (e) {
-                console.log(`Error deleting ${file_path.path}: ${e}`);
-            }
+            // Add MD file to delete queue
+            md_files_delete.push(file_path.path)
+
             conversion_success++;
             return true;
         }
@@ -553,6 +554,7 @@
     };
 
     const dreeOptions = {
+        hash: true,
         extensions: ['md', 'html', 'htm'],
         normalize: true,
     };
@@ -789,6 +791,16 @@
             process.exit(1);
         }
 
+        // Delete markdown files
+        for (let i = 0; i < md_files_delete.length; i++) {
+            // Delete MD file from _work path
+            try {
+                fs.unlinkSync(md_files_delete[i]);
+            } catch (e) {
+                console.log(`Error deleting ${md_files_delete[i]}: ${e}`);
+            }
+        }
+
         // Add book read timing to the hdocbook.json
         hdocbook_config.readingTime = Math.ceil(book_read_time + ((book_read_time / 100) * 10));
         try {

package/hdoc-validate.js

@@ -5,7 +5,6 @@
         cheerio = require('cheerio'),
         dree = require('dree'),
         fs = require('fs'),
-        html2text = require('html-to-text'),
         https = require('https'),
         path = require('path'),
         hdoc = require(path.join(__dirname, 'hdoc-module.js')),
@@ -26,7 +25,8 @@
         messages = {},
         warnings = {},
         errorcount = 0,
-        html_files = [],
+        html_to_validate = [],
+        md_to_validate = [],
         exclude_links = {},
         exclude_spellcheck = {};
 
@@ -45,14 +45,9 @@
         return true;
     };
 
-    const spellcheckHTML = async function (htmlFile, excludes) {
-        const htmlBody = fs.readFileSync(htmlFile.path, 'utf8');
-        const text = html2text.convert(htmlBody, {
-            wordwrap: null,
-            selectors: [
-                { selector: 'code', format: 'skip'}
-              ]
-        });
+    const spellcheckContent = async function (sourceFile, excludes) {
+        const text = fs.readFileSync(sourceFile.path, 'utf8');
+        const source_path = sourceFile.relativePath.replace('.'+sourceFile.extension, '');
         const translate_output = translator.translate(text, spellcheck_options);
         if(Object.keys(translate_output).length){
             for (const key in translate_output) {
@@ -61,10 +56,10 @@
                     for (let i = 0; i < translate_output[key].length; i++) {
                         for (const spelling in translate_output[key][i]) {
                             if (translate_output[key][i].hasOwnProperty(spelling)) {
-                                if (!excludes[htmlFile.relativePath]) {
-                                    errors[htmlFile.relativePath].push(`${error_message} ${spelling} should be ${translate_output[key][i][spelling].details}`);
-                                } else if (!excludes[htmlFile.relativePath].includes(spelling.toLowerCase())) {
-                                    errors[htmlFile.relativePath].push(`${error_message} ${spelling} should be ${translate_output[key][i][spelling].details}`);
+                                if (!excludes[source_path]) {
+                                    errors[sourceFile.relativePath].push(`${error_message} ${spelling} should be ${translate_output[key][i][spelling].details}`);
+                                } else if (!excludes[source_path].includes(spelling.toLowerCase())) {
+                                    errors[sourceFile.relativePath].push(`${error_message} ${spelling} should be ${translate_output[key][i][spelling].details}`);
                                 }
                             }
                         }
@@ -197,7 +192,7 @@
     const dreeOptions = {
         descendants: true,
         depth: 10,
-        extensions: ['htm', 'html'],
+        extensions: ['htm', 'html','md'],
         hash: false,
         normalize: true,
         size: false,
@@ -208,7 +203,11 @@
 
     // File scan callback
     const fileCallback = function (element) {
-        html_files.push(element);
+        if (element.extension.toLowerCase() === 'md') {
+            md_to_validate.push(element);
+        } else {
+            html_to_validate.push(element);
+        }
     };
 
     const isRelativePath = function (source_path, html_path, relative_path) {
@@ -338,32 +337,47 @@
             }
         }
 
+        // Do spellchecking on markdown files
+        let md_files_spellchecked = {};
+        for (let i = 0; i < md_to_validate.length; i++) {
+            // Initiate maps for errors and verbose messages for markdown file
+            errors[md_to_validate[i].relativePath] = [];
+            messages[md_to_validate[i].relativePath] = [];
+            warnings[md_to_validate[i].relativePath] = [];
+
+            await spellcheckContent(md_to_validate[i], exclude_spellcheck);
+            md_files_spellchecked[md_to_validate[i].relativePath.replace('.' + md_to_validate[i].extension, '')] = true;
+        }
+
+        // Perform rest of validation against HTML files
         let listContent = '';
-        for (let i = 0; i < html_files.length; i++) {
+        for (let i = 0; i < html_to_validate.length; i++) {
 
             // Initiate maps for errors and verbose messages for HTML file
-            errors[html_files[i].relativePath] = [];
-            messages[html_files[i].relativePath] = [];
-            warnings[html_files[i].relativePath] = [];
+            errors[html_to_validate[i].relativePath] = [];
+            messages[html_to_validate[i].relativePath] = [];
+            warnings[html_to_validate[i].relativePath] = [];
 
-            // Check for Britishisms
-            await spellcheckHTML(html_files[i], exclude_spellcheck);
+            // Check for British spellings in static HTML content
+            if (!md_files_spellchecked[html_to_validate[i].relativePath.replace('.' + html_to_validate[i].extension, '')]) {
+                await spellcheckContent(html_to_validate[i], exclude_spellcheck);
+            }
 
-            const links = getLinks(html_files[i]);
+            const links = getLinks(html_to_validate[i]);
             if (links.href.length === 0) {
-                messages[html_files[i].relativePath].push('No links found in file');
+                messages[html_to_validate[i].relativePath].push('No links found in file');
             } else {
-                await checkLinks(source_path, html_files[i], links.href, hdocbook_config);
+                await checkLinks(source_path, html_to_validate[i], links.href, hdocbook_config);
             }
             if (links.img.length === 0) {
-                messages[html_files[i].relativePath].push('No images found in file');
+                messages[html_to_validate[i].relativePath].push('No images found in file');
             } else {
-                await checkImages(source_path, html_files[i], links.img);
+                await checkImages(source_path, html_to_validate[i], links.img);
             }
 
             // Build list content for Google 
-            listContent += `/${html_files[i].relativePath.replace(path.extname(html_files[i].relativePath), '')}`;
-            if (i < html_files.length - 1) {
+            listContent += `/${html_to_validate[i].relativePath.replace(path.extname(html_to_validate[i].relativePath), '')}`;
+            if (i < html_to_validate.length - 1) {
                 listContent += '\r\n';
             }
         }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "hdoc-tools",
-  "version": "0.9.7",
+  "version": "0.9.9",
   "description": "Hornbill HDocBook Development Support Tool",
   "main": "hdoc.js",
   "bin": {