hdoc-tools 0.9.29 → 0.9.30

package/hdoc-build-db.js

@@ -79,68 +79,71 @@
 
         if (!book_config.tags) book_config.tags = [];
 
+        let indexPromises = [];
         for (let i = 0; i < index_records.length; i++) {
-            let index_path_name = index_records[i].relative_path.replace('\\', '/');
-            index_path_name = '/' + index_path_name.replace(path.extname(index_records[i].relative_path), '');
+            indexPromises.push(index_records[i]);
+        }
+        await Promise.all(indexPromises.map(async (file) => {
+            let index_path_name = file.relative_path.replace('\\', '/');
+            index_path_name = '/' + index_path_name.replace(path.extname(file.relative_path), '');
             const index_vals = [
                 index_path_name,
                 doc_id,
                 book_config.audience.join(','),
                 book_config.tags.join(','),
-                index_records[i].index_html.fm_props.title,
-                index_records[i].index_html.text,
-                index_records[i].index_html.preview,
+                file.index_html.fm_props.title,
+                file.index_html.text,
+                file.index_html.preview,
                 book_config.productFamily,
-                index_records[i].md5
+                file.md5
             ];
-            const index_response = await  hdoc_index.insert_record(db, 'hdoc_index', db_schema.hdoc_index, index_vals);
+            const index_response = hdoc_index.insert_record(db, 'hdoc_index', db_schema.hdoc_index, index_vals);
             if (!index_response.success) {
-                console.log(`Index record creation failed - ${doc_id}/${index_records[i].index_html.fm_props.title}: ${index_response.error}`);
-                continue;
-            }
-
-            // Now add metadata
-            const meta_vals = [
-                index_path_name,
-                doc_id,
-                index_records[i].metadata.contributor_count,
-                index_records[i].metadata.edit_url,
-                index_records[i].metadata.last_commit,
-                index_records[i].pdf_size
-            ];
-            const meta_response = await hdoc_index.insert_record(db, 'hdoc_meta', db_schema.hdoc_meta, meta_vals);
-            if (!meta_response.success) {
-                console.log(`Index metadata record creation failed - ${doc_id}/${index_response.row_id}/${index_records[i].index_html.fm_props.title}: ${meta_response.error}`);
-                continue;
-            }
-            if (verbose) {
-                console.log(`Inserted index record ${index_response.row_id}: ${doc_id} - ${index_records[i].index_html.fm_props.title}`);
-                console.log(`Inserted index metadata record for index ID: ${meta_response.row_id}`);
-            }
-
-            // Now add contributor records
-            for (let j = 0; j < index_records[i].contributors.length; j++) {
-                const contrib_vals = [
+                console.log(`Index record creation failed - ${doc_id}/${file.index_html.fm_props.title}: ${index_response.error}`);
+            } else {
+                // Now add metadata
+                const meta_vals = [
                     index_path_name,
                     doc_id,
-                    index_records[i].contributors[j].login,
-                    index_records[i].contributors[j].name,
-                    index_records[i].contributors[j].avatar_url,
-                    index_records[i].contributors[j].html_url
+                    file.metadata.contributor_count,
+                    file.metadata.edit_url,
+                    file.metadata.last_commit,
+                    file.pdf_size
                 ];
-                const cont_response = await hdoc_index.insert_record(db, 'hdoc_contributors', db_schema.hdoc_contributors, contrib_vals);
-                if (!cont_response.success) {
-                    console.log(`Index document contributor record creation failed - ${doc_id}/${index_response.row_id}/${index_records[i].index_html.fm_props.title}: ${cont_response.error}`);
-                    continue;
+                const meta_response = await hdoc_index.insert_record(db, 'hdoc_meta', db_schema.hdoc_meta, meta_vals);
+                if (!meta_response.success) {
+                    console.log(`Index metadata record creation failed - ${doc_id}/${index_response.row_id}/${file.index_html.fm_props.title}: ${meta_response.error}`);
+                } else {
+                    if (verbose) {
+                        console.log(`Inserted index record ${index_response.row_id}: ${doc_id} - ${file.index_html.fm_props.title}`);
+                        console.log(`Inserted index metadata record for index ID: ${meta_response.row_id}`);
+                    }
+        
+                    // Now add contributor records
+                    for (let j = 0; j < file.contributors.length; j++) {
+                        const contrib_vals = [
+                            index_path_name,
+                            doc_id,
+                            file.contributors[j].login,
+                            file.contributors[j].name,
+                            file.contributors[j].avatar_url,
+                            file.contributors[j].html_url
+                        ];
+                        const cont_response = await hdoc_index.insert_record(db, 'hdoc_contributors', db_schema.hdoc_contributors, contrib_vals);
+                        if (!cont_response.success) {
+                            console.log(`Index document contributor record creation failed - ${doc_id}/${index_response.row_id}/${file.index_html.fm_props.title}: ${cont_response.error}`);
+                            continue;
+                        }
+                        if (verbose) {
+                            console.log(`Inserted document contributor recordL ${cont_response.row_id}`);
+                        }   
+                    }
+                    response.index_success_count++;
                 }
-                if (verbose) {
-                    console.log(`Inserted document contributor recordL ${cont_response.row_id}`);
-                }   
             }
 
-            response.index_success_count++;
-            
-        }
+        }));
+
         response.success = true;
         console.log(`\nIndex Build Complete: ${response.index_success_count} document index records created.`);
         return response;

package/hdoc-build-pdf.js

@@ -9,20 +9,10 @@
         path = require('path'),
         hdoc = require(path.join(__dirname, 'hdoc-module.js'));
 
-    const dree_options = {
-        extensions: ['css'],
-        normalize: true,
-    };
-
-    let css_files = [],
-        hb_logo = '',
+    let hb_logo = '',
         footer = '',
         header = '';
 
-    const file_callback = function (element) {
-        css_files.push(element.path);
-    };
-
     const get_footer = function (template_path) {
         let footer_content = null;
         try {
@@ -109,9 +99,11 @@
         return html_source;
     };
 
-    exports.generate_pdf = async function (browser, pdf_template_path, pdf_template_content, book_config, html_source, target_file, verbose = false) {
-        if (verbose) console.log(`Generating PDF: ${target_file}`);
+    exports.generate_pdf = async function (browser, pdf_template_path, pdf_template_content, book_config, html_source, target_file, css_templates, verbose = false) {
         let pdf_size = 0;
+    
+        //dree.scan(pdf_template_path, dree_options_css, file_callback_css);
+
         // Cache footer
         if (footer === '') footer = get_footer(pdf_template_path);
 
@@ -135,37 +127,28 @@
 
         html_source = pdf_template_content.replace('{{book_title}}', book_config.title).replace('{{document_content}}', html_source);
 
-        // Create a new page
         const page = await browser.newPage();
 
+        // To reflect CSS used for screens instead of print
+        await page.emulateMediaType('screen');
+
         // Set HTML content from HTML source
         await page.setContent(html_source, {
             waitUntil: 'domcontentloaded'
         });
-
-        // To reflect CSS used for screens instead of print
-        await page.emulateMediaType('screen');
-
-        if (css_files.length === 0) {
-            // Get a list of CSS files
-            dree.scan(pdf_template_path, dree_options, file_callback);
-        }
-
-        for (let i = 0; i < css_files.length; i++) {
-            await page.addStyleTag({
-                path: css_files[i]
-            });
+        for (let i = 0; i < css_templates.length; i++) {
+            try {
+                await page.addStyleTag({
+                    content: css_templates[i]
+                });
+            } catch (e) {
+                console.log(`Error applying template for [${target_file}]: ${e}`);
+            }
         }
 
         try {
             const pdf_gen = await page.pdf({
                 path: target_file,
-                margin: {
-                    top: '30px',
-                    right: '35px',
-                    bottom: '80px',
-                    left: '35px'
-                },
                 printBackground: true,
                 format: 'A4',
                 displayHeaderFooter: true,
@@ -178,7 +161,9 @@
                     left: "30px"
                 }
             });
-            if (verbose) console.log('PDF generation success.');
+            let currdate = new Date;
+            let datetime = currdate.toISOString();
+            if (verbose) console.log(`[${datetime}] PDF generation success: ${target_file}`);
 
             pdf_size = pdf_gen.byteLength;
         } catch (err) {

package/hdoc-build.js

@@ -1,3 +1,5 @@
+const { files } = require('jszip');
+
 (function () {
     'use strict';
 
@@ -33,6 +35,7 @@
         conversion_attempted = 0,
         conversion_success = 0,
         conversion_failed = 0,
+        css_templates = [],
         doc_header_template = '',
         doc_header_template_non_git = '',
         pdf_created = 0,
@@ -54,6 +57,7 @@
         work_path_content = '',
         verbose = false;
 
+
     const pdf_path_excluded = function (relative_path) {
         if (!hdocbook_project.pdfGeneration || hdocbook_project.pdfGeneration.exclude_paths === undefined) {
             return false;
@@ -241,7 +245,7 @@
 
                 // Generate PDF file from HTML
                 const pdf_file_path = file_path.path.replace(path.extname(file_path.path), '.pdf');
-                pdf_size = await hdoc_build_pdf.generate_pdf(browser, pdf_template_path, pdf_template, hdocbook_config, pdf_txt, pdf_file_path, verbose);
+                pdf_size = await hdoc_build_pdf.generate_pdf(browser, pdf_template_path, pdf_template, hdocbook_config, pdf_txt, pdf_file_path, css_templates, verbose);
             }
             if (pdf_size > 0) pdf_created++;
 
@@ -452,7 +456,7 @@
 
                 // Generate PDF file from HTML
                 const pdf_file_path = file_path.path.replace(path.extname(file_path.path), '.pdf');
-                pdf_size = await hdoc_build_pdf.generate_pdf(browser, pdf_template_path, pdf_template, hdocbook_config, pdf_txt, pdf_file_path, verbose);
+                pdf_size = await hdoc_build_pdf.generate_pdf(browser, pdf_template_path, pdf_template, hdocbook_config, pdf_txt, pdf_file_path, css_templates, verbose);
             }
             if (pdf_size > 0) pdf_created++;
             html_txt = `${fm_header}\n${doc_header}\n${html_txt}`;
@@ -606,7 +610,9 @@
         if (!validate && hdocbook_project.pdfGeneration !== undefined && hdocbook_project.pdfGeneration.enable !== undefined) {
             pdf_enable = hdocbook_project.pdfGeneration.enable;
         }
-
+    
+        console.log(`Loading hdocbook config...`);
+        
         const book_path = path.join(source_path, doc_id),
             hdocbook_path = path.join(book_path, 'hdocbook.json'),
             work_path = path.join(source_path, '_work'),
@@ -624,6 +630,20 @@
 
         if (hdocbook_config.publicSource.endsWith('.git')) hdocbook_config.publicSource = hdocbook_config.publicSource.substring(0, hdocbook_config.publicSource.length - 4);
 
+        console.log('Caching CSS for PDF generation...');
+        const css_files = [
+            path.join(pdf_template_path, 'css', 'custom-block.css'),
+            path.join(pdf_template_path, 'css', 'hdocs-pdf.css'),
+            path.join(pdf_template_path, 'css', 'vars.css')
+        ];
+        for (let i = 0; i < css_files.length; i++) {
+            try {
+                css_templates.push(fs.readFileSync(css_files[i], 'utf8'));
+            } catch (e) {
+                console.log(`Error reading file [${css_files[i]}]: ${e}`);
+            }
+        }
+        
         console.log(`Building: ${doc_id} v${hdocbook_config.version}...\n`);
 
         // Make _work folder to copy everything into
@@ -637,6 +657,7 @@
         fs.mkdirSync(work_path);
 
         // Copy files from book into _work-doc_id folder
+        console.log(`Copying content into work folder...`);
         try {
             fs.copySync(path.join(source_path, doc_id), work_path_content);
         } catch (e) {
@@ -644,7 +665,9 @@
             process.exit(1);
         }
 
-        // Ccreate MD5 hash of content before build
+        // Create MD5 hash of content before build
+        console.log(`Creating Hash...`);
+        
         dree.scan(work_path_content, md5DreeOptions, hash_callback);
         let concat_hash = '|';
         for (let i = 0; i < built_file_hashes.length; i++) {
@@ -660,13 +683,14 @@
         const checksum_path = path.join(work_path_content, 'checksum.md5');
         try {
             fs.writeFileSync(checksum_path, hash);
-            console.log('\nHash file creation success:', checksum_path);
+            console.log('Hash file creation success:', checksum_path);
         } catch (e) {
             console.log('\nError creating', checksum_path, ':', e);
             process.exit(1);
         }
 
         // Load document header templates
+        console.log(`Loading templates...`);
         try {
             doc_header_template = fs.readFileSync(doc_header_template_path, 'utf8');
             doc_header_template_non_git = fs.readFileSync(non_git_doc_header_template_path, 'utf8');
@@ -686,27 +710,36 @@
                 process.exit(1);
             }
         }
-        console.log('Processing navigation breadcrumbs...\n');
+        console.log(`Processing navigation breadcrumbs...`);
         bc = hdoc.build_breadcrumbs(hdocbook_config.navigation.items);
 
-        console.log('Processing content...');
+        console.log(`Processing content...`);
         // Get a list of MD files in work_path
         dree.scan(work_path, dreeOptions, build_file_callback);
 
         if (pdf_enable) {
-            // Create a Chromium browser instance to generate PDFs with
+            // Create a Chromium browser instance generate PDFs with
             browser = await puppeteer.launch({headless: 'new'});
         }
 
         // Work through MD files and convert to HTML
+        let mdPromiseArray = [];
         for (let i = 0; i < md_files.length; i++) {
-            await transform_markdown_and_save_html(md_files[i]);
+            mdPromiseArray.push(md_files[i]);
         }
+        await Promise.all(mdPromiseArray.map(async (file) => {
+            await transform_markdown_and_save_html(file);
+        }));
 
         // Work through Static HTML files and add Frontmatter tags
+        let htmlPromiseArray = [];
         for (let i = 0; i < static_html_files.length; i++) {
-            await transform_static_html(static_html_files[i]);
+            htmlPromiseArray.push(static_html_files[i]);
         }
+        await Promise.all(htmlPromiseArray.map(async (file) => {
+            await transform_static_html(file);
+        }));
+
 
         if (pdf_enable) {
             // Close the Chromium browser instance
@@ -734,14 +767,17 @@
         }
 
         // Delete markdown files
+        console.log(`Performing Markdown Cleanup`);
+
+        let filePromiseArray = [];
         for (let i = 0; i < md_files_delete.length; i++) {
-            // Delete MD file from _work path
-            try {
-                fs.unlinkSync(md_files_delete[i]);
-            } catch (e) {
-                console.log(`Error deleting ${md_files_delete[i]}: ${e}`);
-            }
+            filePromiseArray.push(md_files_delete[i]);
         }
+        await Promise.all(filePromiseArray.map(async (file) => {
+            fs.unlink(file, (err => {
+                if (err) console.log(`Error deleting ${file}: ${e}`);
+            }));
+        }));
     
         // Add book read timing to the hdocbook.json
         hdocbook_config.readingTime = Math.ceil(book_read_time + ((book_read_time / 100) * 10));
@@ -756,6 +792,7 @@
 
         // Build the index
         // Create the DB and tables
+        console.log(`Building the Index`);
         let db = hdoc_build_db.create_db(work_path, doc_id);
         if (db.error && db.error !== null) {
             console.log(db.error);
@@ -767,6 +804,9 @@
             console.log(index.error);
             process.exit(1);
         }
+        if (verbose) {
+            console.log(`Index Build Complete`);
+        }
 
         if (!validate) {
             try {

package/hdoc-db.js

@@ -25,7 +25,7 @@
         }
     };
 
-    exports.insert_record = async function (db, table, columns, values) {
+    exports.insert_record = function (db, table, columns, values) {
         let response = {
             success: false,
             row_id: 0,

package/hdoc-validate.js

@@ -405,51 +405,59 @@
 
         // Do spellchecking on markdown files
         let md_files_spellchecked = {};
+        let mdPromiseArray = [];
         for (let i = 0; i < md_to_validate.length; i++) {
-            // Initiate maps for errors and verbose messages for markdown file
-            errors[md_to_validate[i].relativePath] = [];
-            messages[md_to_validate[i].relativePath] = [];
-            warnings[md_to_validate[i].relativePath] = [];
-
-            await spellcheckContent(md_to_validate[i], exclude_spellcheck);
-            md_files_spellchecked[md_to_validate[i].relativePath.replace('.' + md_to_validate[i].extension, '')] = true;
+            mdPromiseArray.push(md_to_validate[i]);
         }
+        await Promise.all(mdPromiseArray.map(async (file) => {
+            //await transform_markdown_and_save_html(file);
+            // Initiate maps for errors and verbose messages for markdown file
+            errors[file.relativePath] = [];
+            messages[file.relativePath] = [];
+            warnings[file.relativePath] = [];
+    
+            await spellcheckContent(file, exclude_spellcheck);
+            md_files_spellchecked[file.relativePath.replace('.' + file.extension, '')] = true;
+        }));
 
         // Perform rest of validation against HTML files
         let listContent = '';
+        let htmlPromiseArray = [];
         for (let i = 0; i < html_to_validate.length; i++) {
-
+            htmlPromiseArray.push(html_to_validate[i]);
+        }
+        await Promise.all(mdPromiseArray.map(async (file) => {
             // Initiate maps for errors and verbose messages for HTML file
-            errors[html_to_validate[i].relativePath] = [];
-            messages[html_to_validate[i].relativePath] = [];
-            warnings[html_to_validate[i].relativePath] = [];
+            errors[file.relativePath] = [];
+            messages[file.relativePath] = [];
+            warnings[file.relativePath] = [];
 
             // Check for British spellings in static HTML content
-            if (!md_files_spellchecked[html_to_validate[i].relativePath.replace('.' + html_to_validate[i].extension, '')]) {
-                await spellcheckContent(html_to_validate[i], exclude_spellcheck);
+            if (!md_files_spellchecked[file.relativePath.replace('.' + file.extension, '')]) {
+                await spellcheckContent(file, exclude_spellcheck);
             }
 
-            const links = getLinks(html_to_validate[i]);
+            const links = getLinks(file);
             if (links.href.length === 0) {
-                messages[html_to_validate[i].relativePath].push('No links found in file');
+                messages[file.relativePath].push('No links found in file');
             } else {
-                await checkLinks(source_path, html_to_validate[i], links.href, hdocbook_config);
+                await checkLinks(source_path, file, links.href, hdocbook_config);
             }
             if (links.img.length === 0) {
-                messages[html_to_validate[i].relativePath].push('No images found in file');
+                messages[file.relativePath].push('No images found in file');
             } else {
-                await checkImages(source_path, html_to_validate[i], links.img);
+                await checkImages(source_path, file, links.img);
             }
 
             // Check for multiple H1 tags
-            await checkTags(html_to_validate[i]);
+            await checkTags(file);
             
             // Build list content for Google 
-            listContent += `/${html_to_validate[i].relativePath.replace(path.extname(html_to_validate[i].relativePath), '')}`;
-            if (i < html_to_validate.length - 1) {
-                listContent += '\r\n';
-            }
-        }
+            listContent += `/${file.relativePath.replace(path.extname(file.relativePath), '')}`;
+            listContent += '\r\n';
+        }));
+
+
         try {
             // Write list
             const listFile = path.join(source_path, doc_id, 'links.txt');

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "hdoc-tools",
-  "version": "0.9.29",
+  "version": "0.9.30",
   "description": "Hornbill HDocBook Development Support Tool",
   "main": "hdoc.js",
   "bin": {