hdoc-tools 0.8.22 → 0.8.24

package/hdoc-build-db.js

@@ -0,0 +1,141 @@
+(function () {
+    'use strict';
+
+    const   path = require('path'),
+            hdoc_index = require(path.join(__dirname, 'hdoc-db.js')),
+            database = require('better-sqlite3');
+
+    const db_schema = {
+        hdoc_index: [
+            'relative_url UNINDEXED',
+            'book_id',
+            'book_audience UNINDEXED',
+            'book_tags',
+            'doc_title',
+            'doc_content',
+            'doc_preview UNINDEXED',
+            'doc_family_id'
+        ],
+        hdoc_meta: [
+            'doc_id INTEGER',
+            'contributor_count INTEGER UNINDEXED',
+            'edit_url UNINDEXED',
+            'last_commit UNINDEXED'
+        ],
+        hdoc_contributors: [
+            'doc_id INTEGER',
+            'login',
+            'name',
+            'avatar UNINDEXED',
+            'url UNINDEXED'
+        ]
+    };
+
+    exports.create_table = function (db, table, cols, virtual, fts5) {
+        const table_created = hdoc_index.create_table(db, table, cols, virtual, fts5);
+        if (table_created !== null) {
+            return `\nError creating table: ${table_created}`;
+        } else {
+            console.log(`\nTable created: ${table}`);
+        }
+        return null;
+    };
+
+    exports.create_db = function (db_path, doc_id) {
+        let response = {
+            error: null,
+            db: null
+        };
+        console.log('Performing SQlite index creation...');
+        let db_name = path.join(db_path, doc_id, 'index.db');
+        response.db = new database(db_name);
+        response.db.pragma('encoding="UTF-8"');
+        console.log(`\nDatabase created: ${db_name}`);
+
+        // Create tables
+        for (const key in db_schema) {
+            if (db_schema.hasOwnProperty(key)) {
+                const virtual = key === 'hdoc_index' ? true : false;
+                const fts5 = key === 'hdoc_index' ? true : false;
+                const table_created_error = this.create_table(response.db, key, db_schema[key], virtual, fts5);
+                if (table_created_error !== null) {
+                    console.log(table_created_error);
+                }
+            }
+        }
+        return response;
+    };
+
+    exports.populate_index = function (db, doc_id, book_config, index_records, verbose = false) {
+        let response = {
+            success: false,
+            index_success_count: 0,
+            error: ''
+        };
+
+        if (!book_config.tags) book_config.tags = [];
+
+        for (let i = 0; i < index_records.length; i++) {
+            let index_path_name = index_records[i].relative_path.replace('\\', '/').replace(`${doc_id}/`, '');
+            index_path_name = index_path_name.replace(path.extname(index_records[i].relative_path), '');
+            const index_vals = [
+                index_path_name,
+                doc_id,
+                book_config.audience.join(','),
+                book_config.tags.join(','),
+                index_records[i].index_html.fm_props.title,
+                index_records[i].index_html.text,
+                index_records[i].index_html.preview,
+                book_config.productFamily
+            ];
+            const index_response = hdoc_index.insert_record(db, 'hdoc_index', db_schema.hdoc_index, index_vals);
+            if (!index_response.success) {
+                console.log(`Index record creation failed - ${doc_id}/${index_records[i].index_html.fm_props.title}: ${index_response.error}`);
+                continue;
+            }
+
+            // Now add metadata
+            const meta_vals = [
+                parseInt(index_response.row_id, 10),
+                index_records[i].metadata.contributor_count,
+                index_records[i].metadata.edit_url,
+                index_records[i].metadata.last_commit
+            ];
+            const meta_response = hdoc_index.insert_record(db, 'hdoc_meta', db_schema.hdoc_meta, meta_vals);
+            if (!meta_response.success) {
+                console.log(`Index metadata record creation failed - ${doc_id}/${index_response.row_id}/${index_records[i].index_html.fm_props.title}: ${meta_response.error}`);
+                continue;
+            }
+            if (verbose) {
+                console.log(`Inserted index record ${index_response.row_id}: ${doc_id} - ${index_records[i].index_html.fm_props.title}`);
+                console.log(`Inserted index metadata record for index ID: ${meta_response.row_id}`);
+            }
+
+            // Now add contributor records
+            for (let j = 0; j < index_records[i].contributors.length; j++) {
+                const contrib_vals = [
+                    parseInt(index_response.row_id, 10),
+                    index_records[i].contributors[j].login,
+                    index_records[i].contributors[j].name,
+                    index_records[i].contributors[j].avatar_url,
+                    index_records[i].contributors[j].html_url
+                ];
+                const cont_response = hdoc_index.insert_record(db, 'hdoc_contributors', db_schema.hdoc_contributors, contrib_vals);
+                if (!cont_response.success) {
+                    console.log(`Index document contributor record creation failed - ${doc_id}/${index_response.row_id}/${index_records[i].index_html.fm_props.title}: ${cont_response.error}`);
+                    continue;
+                }
+                if (verbose) {
+                    console.log(`Inserted document contributor recordL ${cont_response.row_id}`);
+                }   
+            }
+
+            response.index_success_count++;
+            
+        }
+        response.success = true;
+        console.log(`\nIndex Build Complete:`);
+        console.log(`${response.index_success_count} document index records created.`);
+        return response;
+    };
+})();

package/hdoc-build-pdf.js

@@ -0,0 +1,169 @@
+(function () {
+    'use strict';
+
+    const cheerio = require('cheerio'),
+        dree = require('dree'),
+        fs = require('fs-extra'),
+        mime = require('mime-types'),
+        path = require('path'),
+        puppeteer = require('puppeteer'),
+        request = require('sync-request'),
+        hdoc = require(path.join(__dirname, 'hdoc-module.js'));
+
+    const dree_options = {
+        extensions: ['css'],
+        normalize: true,
+    };
+
+    let css_files = [],
+        hb_logo = '',
+        header = '',
+        footer = '';
+
+    const file_callback = function (element) {
+        css_files.push(element.path);
+    };
+
+    const get_footer = function (template_path) {
+        let footer_content = null;
+        try {
+            footer_content = fs.readFileSync(path.join(template_path, 'template-footer.html'), 'utf8');
+        } catch (err) {
+            console.log(`Error loading template: ${err}`);
+        }
+        return footer_content;
+    };
+
+    exports.process_images = async function (file_path, html_source, verbose) {
+        const book_work_root = file_path.path.replace(file_path.relativePath, '');
+        if (verbose) console.log('Parsing img tags from HTML source');
+
+        // Use cheerio to parse html
+        const $ = cheerio.load(html_source);
+        
+        // Get iFrames from HTML, to replace with a tags
+        let iframes = [];
+        const iframe_html = $('iframe').map(function () {
+            const response = {
+                html: $.html(this),
+                src: $(this).attr('src'),
+                title: $(this).attr('title') ? $(this).attr('title') : 'No Link Title Provided'
+            };
+            return response;
+        }).get();
+        iframes.push(...iframe_html);
+        for (let i = 0; i < iframes.length; i++) {
+            const link = `<p><a href="${iframes[i].src}">${iframes[i].title}</a></p>`;
+            const regex = new RegExp(`<iframe.*src="${iframes[i].src.replace('/', '\\/')}".*</iframe>`);
+            html_source = html_source.replace(regex, link);
+        }
+        
+        // Get image links from HTML, to embed into the pdf
+        let imgs = [];
+        const srcs = $('img').map(function (i) {
+            return $(this).attr('src');
+        }).get();
+        imgs.push(...srcs);
+        for (let i = 0; i < imgs.length; i++) {
+            if (hdoc.valid_url(imgs[i]).valid) {
+                // External Link
+                const file_response = request('GET', imgs[i]);
+                if (file_response.statusCode === 200) {
+                    const image_buffer = file_response.getBody();
+                    const mime_type = mime.lookup(imgs[i]);
+                    let image_b64 = image_buffer.toString("base64");
+                    image_b64 = `data:${mime_type};base64,${image_b64}`;
+                    html_source = html_source.replace(imgs[i], image_b64);
+                } else {
+                    throw `Unexpected Status ${file_response.statusCode}`;
+                }
+            } else {
+                // Internal link
+                const image_path = path.join(book_work_root, imgs[i].replace('_books/', ''));
+                try {
+                    const image_buffer = fs.readFileSync(image_path);
+                    const mime_type = mime.lookup(image_path);
+                    let image_b64 = image_buffer.toString("base64");
+                    image_b64 = `data:${mime_type};base64,${image_b64}`;
+                    html_source = html_source.replace(imgs[i], image_b64);
+                } catch (err) {
+                    console.log('Error reading image from HTML source [', image_path, '] - ', err);
+                    return null;
+                }
+            }
+        }
+
+        return html_source;
+    };
+
+    exports.generate_pdf = async function (pdf_template_path, pdf_template_content, book_config, html_source, target_file, verbose = false) {
+        if (verbose) console.log(`Generating PDF: ${target_file}`);
+
+        // Cache footer
+        if (footer === '') footer = get_footer(pdf_template_path);
+
+        // Read svg logo file into buffer, convert to B64 string 
+        if (hb_logo === '') {
+            const hb_logo_path = path.join(pdf_template_path, 'images', 'hornbill-logo-full.svg');
+            try {
+                const hb_logo_file_buffer = fs.readFileSync(hb_logo_path);
+                hb_logo = hb_logo_file_buffer.toString("base64");
+                hb_logo = `data:image/svg+xml;base64,${hb_logo}`;
+            } catch (err) {
+                console.log('Error reading logo from template:', err);
+                return;
+            }
+        }
+
+        html_source = pdf_template_content.replace('{{book_title}}', book_config.title).replace('{{document_content}}', html_source).replace('{{hb_logo}}', hb_logo);
+
+        // Create a browser instance
+        const browser = await puppeteer.launch();
+
+        // Create a new page
+        const page = await browser.newPage();
+
+        // Set HTML content from HTML source
+        await page.setContent(html_source, {
+            waitUntil: 'domcontentloaded'
+        });
+
+        // To reflect CSS used for screens instead of print
+        await page.emulateMediaType('screen');
+
+        if (css_files.length === 0) {
+            // Get a list of CSS files
+            dree.scan(pdf_template_path, dree_options, file_callback);
+        }
+
+        for (let i = 0; i < css_files.length; i++) {
+            await page.addStyleTag({
+                path: css_files[i]
+            });
+        }
+
+        try {
+            await page.pdf({
+                path: target_file,
+                margin: {
+                    top: '30px',
+                    right: '35px',
+                    bottom: '80px',
+                    left: '35px'
+                },
+                printBackground: true,
+                format: 'A4',
+                displayHeaderFooter: true,
+                headerTemplate: '<div />',
+                footerTemplate: footer
+            });
+            if (verbose) console.log('PDF generation success.');
+
+        } catch (err) {
+            console.log(`Error generating PDF ${target_file} - ${err}`);
+        }
+
+        // Close the browser instance
+        await browser.close();
+    };
+})();

package/hdoc-build.js

@@ -1,3 +1,7 @@
+const {
+    verbose
+} = require('sqlite3');
+
 (function () {
     'use strict';
 
@@ -10,12 +14,17 @@
         validate = require(path.join(__dirname, 'hdoc-validate.js')),
         hdoc = require(path.join(__dirname, 'hdoc-module.js')),
         hdoc_build_db = require(path.join(__dirname, 'hdoc-build-db.js')),
+        hdoc_build_pdf = require(path.join(__dirname, 'hdoc-build-pdf.js')),
         hdoc_index = require(path.join(__dirname, 'hdoc-db.js')),
         zipper = require('zip-local');
 
     const h_tags_to_search = ['h1', 'h2', 'h3'],
         doc_header_template_path = path.join(__dirname, 'templates', 'doc-header.html'),
-        non_git_doc_header_template_path = path.join(__dirname, 'templates', 'doc-header-non-git.html');
+        non_git_doc_header_template_path = path.join(__dirname, 'templates', 'doc-header-non-git.html'),
+        pdf_header_template_path = path.join(__dirname, 'templates', 'pdf-header.html'),
+        non_git_pdf_header_template_path = path.join(__dirname, 'templates', 'pdf-header-non-git.html'),
+        pdf_template_path = path.join(__dirname, 'templates', 'pdf'),
+        pdf_template_file_path = path.join(pdf_template_path, 'template.html');
 
     let bc = {}, // Breadcrumbs map
         book_read_time = 0,
@@ -25,6 +34,9 @@
         conversion_failed = 0,
         doc_header_template = '',
         doc_header_template_non_git = '',
+        pdf_header_template = '',
+        pdf_header_template_non_git = '',
+        pdf_template = '',
         doc_id = '',
         git_token = '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', // Github fine-grained personal access token that has minimum read-only access to Hornbill Docs org repo content
         hdocbook_config = {},
@@ -37,7 +49,7 @@
         static_html_files = [],
         work_path_content = '';
 
-    const transform_static_html = function (file_path) {
+    const transform_static_html = async function (file_path) {
         if (fs.existsSync(file_path.path)) {
             // Load HTML file
             let html_txt = fs.readFileSync(file_path.path, 'utf8');
@@ -211,7 +223,7 @@
         }
     };
 
-    const transform_markdown_and_save_html = function (file_path) {
+    const transform_markdown_and_save_html = async function (file_path, verbose) {
         conversion_attempted++;
         if (fs.existsSync(file_path.path)) {
             // Load markdown file
@@ -372,17 +384,24 @@
             fm_header += ']]-->';
 
             let doc_header = '';
+            let pdf_header = '';
             if (hdocbook_config.publicSource.includes('github.com/Hornbill-Docs')) {
                 // Build doc header from template and frontmatter tags
                 doc_header = process_doc_header(fm_headers, file_path.relativePath, doc_header_template);
+                pdf_header = process_doc_header(fm_headers, file_path.relativePath, pdf_header_template);
             } else {
                 // Build doc header from template and frontmatter tags
                 doc_header = process_doc_header(fm_headers, file_path.relativePath, doc_header_template_non_git);
+                pdf_header = process_doc_header(fm_headers, file_path.relativePath, pdf_header_template_non_git);
             }
-            /*
-            const pdf_html = `${fm_header}\n${doc_header}\n${html_txt}`;
-            const pdf_file = file_path.path.replace(path.extname(file_path.path), '.pdf');
-            */
+
+            let pdf_txt = await hdoc_build_pdf.process_images(file_path, html_txt);
+            pdf_txt = `${pdf_header}\n${pdf_txt}`;
+
+            // Generate PDF file from HTML
+            const pdf_file_path = file_path.path.replace(path.extname(file_path.path), '.pdf');
+            await hdoc_build_pdf.generate_pdf(pdf_template_path, pdf_template, hdocbook_config, pdf_txt, pdf_file_path, verbose);
+            
             html_txt = `${fm_header}\n${doc_header}\n${html_txt}`;
 
             // Save HTML into HTML file
@@ -603,7 +622,7 @@
         //console.log(JSON.stringify(bc, null, 2));
     };
 
-    exports.run = function (source_path, verbose, github_api_token) {
+    exports.run = async function (source_path, verbose, github_api_token) {
         if (github_api_token !== '') {
             git_token = github_api_token;
         }
@@ -658,25 +677,36 @@
         try {
             doc_header_template = fs.readFileSync(doc_header_template_path, 'utf8');
             doc_header_template_non_git = fs.readFileSync(non_git_doc_header_template_path, 'utf8');
+            pdf_header_template = fs.readFileSync(pdf_header_template_path, 'utf8');
+            pdf_header_template_non_git = fs.readFileSync(non_git_pdf_header_template_path, 'utf8');
         } catch (err) {
             console.log(`Error reading document header template: ${err}`);
             process.exit(1);
         }
 
+        // Load PDF templates
+        try {
+            pdf_template = fs.readFileSync(pdf_template_file_path, 'utf8');
+        } catch (err) {
+            console.log(`Error reading PDF template: ${err}`);
+            process.exit(1);
+        }
+
         build_breadcrumbs(hdocbook_config.navigation.items);
 
+        console.log('Processing content...');
         // Get a list of MD files in work_path
         dree.scan(work_path, dreeOptions, build_file_callback);
 
         // Work through MD files and convert to HTML
-        md_files.forEach(function (md_file) {
-            transform_markdown_and_save_html(md_file);
-        });
+        for (let i = 0; i < md_files.length; i++) {
+            await transform_markdown_and_save_html(md_files[i], verbose);
+        }
 
         // Work through Static HTML files and add Frontmatter tags
-        static_html_files.forEach(function (static_html_file) {
-            transform_static_html(static_html_file);
-        });
+        for (let i = 0; i < static_html_files.length; i++) {
+            await transform_static_html(static_html_files[i]);
+        }
 
         // Output to console
         console.log(`                MD files found: ${conversion_attempted}`);

package/hdoc-init.js

@@ -3,7 +3,6 @@
 
     // Required modules
     const prompt = require('prompt');
-    //const fs = require('fs');
     const fs = require('fs-extra');
     const path = require('path');
 

package/hdoc-validate.js

@@ -1,5 +1,3 @@
-const parseLinkDestination = require('markdown-it/lib/helpers/parse_link_destination');
-
 (function () {
     'use strict';
 

package/hdoc.js

@@ -1,6 +1,6 @@
 #!/usr/bin/env node
 
-(function () {
+(async function () {
    'use strict';
 
    const preRun = require('./validateNodeVer.js'),
@@ -71,7 +71,7 @@
       server.run(ui_path, source_path);
    } else if (command == 'build') {
       const builder = require(path.join(__dirname, 'hdoc-build.js'));
-      builder.run(source_path, verbose, git_token);
+      await builder.run(source_path, verbose, git_token);
    } else if (command == 'stats') {
       const stats = require(path.join(__dirname, 'hdoc-stats.js'));
       stats.run(ui_path, source_path, verbose);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "hdoc-tools",
-  "version": "0.8.22",
+  "version": "0.8.24",
   "description": "Hornbill HDocBook Development Support Tool",
   "main": "hdoc.js",
   "bin": {
@@ -10,6 +10,8 @@
     "hdoc.js",
     "hdoc-db.js",
     "hdoc-build.js",
+    "hdoc-build-db.js",
+    "hdoc-build-pdf.js",
     "hdoc-help.js",
     "hdoc-init.js",
     "hdoc-module.js",
@@ -42,9 +44,10 @@
     "markdown-it": "^13.0.1",
     "markdown-it-container": "^3.0.0",
     "markdown-it-front-matter": "^0.2.3",
+    "mime-types": "^2.1.35",
     "multer": "^1.4.5-lts.1",
-    "pdf-creator-node": "^2.3.5",
     "prompt": "^1.3.0",
+    "puppeteer": "^19.6.0",
     "stream": "0.0.2",
     "sync-request": "^6.1.0",
     "words-count": "^2.0.2",

package/templates/pdf/css/custom-block.css

@@ -0,0 +1,90 @@
+.custom-block {
+  border: 1px solid transparent;
+  border-radius: 8px;
+  padding: 16px 16px 8px;
+  line-height: 24px;
+  font-size: var(--htl-default-font-size);
+  color: var(--htl-c-text-2);
+}
+
+.custom-block.info {
+  border-color: var(--htl-custom-block-info-border);
+  color: var(--htl-custom-block-info-text);
+  background-color: var(--htl-custom-block-info-bg);
+}
+
+.custom-block.info code {
+  background-color: var(--htl-custom-block-info-code-bg);
+}
+
+.custom-block.tip {
+  border-color: var(--htl-custom-block-tip-border);
+  color: var(--htl-custom-block-tip-text);
+  background-color: var(--htl-custom-block-tip-bg);
+}
+
+.custom-block.tip code {
+  background-color: var(--htl-custom-block-tip-code-bg);
+}
+
+.custom-block.warning {
+  border-color: var(--htl-custom-block-warning-border);
+  color: var(--htl-custom-block-warning-text);
+  background-color: var(--htl-custom-block-warning-bg);
+}
+
+.custom-block.warning code {
+  background-color: var(--htl-custom-block-warning-code-bg);
+}
+
+.custom-block.danger {
+  border-color: var(--htl-custom-block-danger-border);
+  color: var(--htl-custom-block-danger-text);
+  background-color: var(--htl-custom-block-danger-bg);
+}
+
+.custom-block.danger code {
+  background-color: var(--htl-custom-block-danger-code-bg);
+}
+
+.custom-block.details {
+  border-color: var(--htl-custom-block-details-border);
+  color: var(--htl-custom-block-details-text);
+  background-color: var(--htl-custom-block-details-bg);
+}
+
+.custom-block.details code {
+  background-color: var(--htl-custom-block-details-code-bg);
+}
+
+.custom-block-title {
+  font-weight: 700;
+}
+
+.custom-block p + p {
+  margin: 8px 0;
+}
+
+.custom-block.details summary {
+  margin: 0 0 8px;
+  font-weight: 700;
+}
+
+.custom-block.details summary + p {
+  margin: 8px 0;
+}
+
+.custom-block a {
+  color: inherit;
+  font-weight: 600;
+  text-decoration: underline;
+  transition: opacity 0.25s;
+}
+
+.custom-block a:hover {
+  opacity: 0.6;
+}
+
+.custom-block code {
+  font-size: var(--htl-custom-block-code-font-size);
+}