hdoc-tools 0.7.26 → 0.8.0

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2022 Hornbill Docs
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/custom_modules/tips.js

@@ -1,69 +1,86 @@
-'use strict';
+(function () {
+    'use strict';
 
-// GS: based on markdown-it-tips extension but customized for our specific needs
+    // GS: based on markdown-it-tips extension but customized for our specific needs
 
-var container = require('markdown-it-container');
+    var container = require('markdown-it-container');
 
-module.exports = function md_tip_plugin(md, options) {
-	var containerOpenCount = 0;
-	var links = options ? options.links : true;
-	init();
-	return;
+    module.exports = function md_tip_plugin(md, options) {
+        var containerOpenCount = 0;
+        var links = options ? options.links : true;
+        init();
+        return;
 
-    function icon_from_type(name) {
-        switch (name) {
-            case "note": return { class: "bi bi-exclamation-circle", title: "Note" };
-            case "tip": return { class: "bi bi-lightbulb", title: "Tip" };
-            case "important": return { class: "bi bi-megaphone", title: "Important" };
-            case "caution": return { class: "bi bi-x-circle", title: "Caution" };
-            case "warning": return { class: "bi bi-exclamation-triangle", title: "Warning" };
+        function icon_from_type(name) {
+            switch (name) {
+                case "note":
+                    return {
+                        class: "bi bi-exclamation-circle", title: "Note"
+                    };
+                case "tip":
+                    return {
+                        class: "bi bi-lightbulb", title: "Tip"
+                    };
+                case "important":
+                    return {
+                        class: "bi bi-megaphone", title: "Important"
+                    };
+                case "caution":
+                    return {
+                        class: "bi bi-x-circle", title: "Caution"
+                    };
+                case "warning":
+                    return {
+                        class: "bi bi-exclamation-triangle", title: "Warning"
+                    };
+            }
+        }
+
+        function setupContainer(name) {
+            md.use(container, name, {
+                render: function (tokens, idx) {
+                    if (tokens[idx].nesting === 1) {
+                        containerOpenCount += 1;
+                        let inf = icon_from_type(name);
+                        return '<div class="hdoc-alert alert-icon-' + name + '">\n<p class="hdoc-alert-title"><span class="' + inf.class + '"></span>' + inf.title + '</p>\n';
+                    } else {
+                        containerOpenCount -= 1;
+                        return '</div>\n';
+                    }
+                }
+            });
+        }
+
+        function isContainerOpen() {
+            return containerOpenCount > 0;
+        }
+
+        function selfRender(tokens, idx, options, env, self) {
+            return self.renderToken(tokens, idx, options);
         }
-    }
 
-	function setupContainer(name) { 
-        md.use(container, name, {
-            render: function (tokens, idx) {
-                if (tokens[idx].nesting === 1) {
-                    containerOpenCount += 1;
-                    let inf = icon_from_type(name);
-                    return '<div class="hdoc-alert alert-icon-' + name + '">\n<p class="hdoc-alert-title"><span class="' + inf.class + '"></span>' + inf.title + '</p>\n';
-                } else {
-                    containerOpenCount -= 1;
-                    return '</div>\n';
+        function setupLinks() {
+            var defaultRender = md.renderer.rules.link_open || selfRender;
+
+            md.renderer.rules.link_open = function (tokens, idx, options, env, self) {
+                if (isContainerOpen()) {
+                    tokens[idx].attrPush(['class', 'md-tip-link']);
                 }
+
+                return defaultRender(tokens, idx, options, env, self);
+            };
+        }
+
+        function init() {
+            setupContainer('note');
+            setupContainer('tip');
+            setupContainer('important');
+            setupContainer('caution');
+            setupContainer('warning');
+
+            if (links) {
+                setupLinks();
             }
-        });
-    }
-	
-    function isContainerOpen() {
-        return containerOpenCount > 0;
-    }
-	
-    function selfRender(tokens, idx, options, env, self) {
-        return self.renderToken(tokens, idx, options);
-    }
-    
-	function setupLinks() {
-		var defaultRender = md.renderer.rules.link_open || selfRender;
-		
-		md.renderer.rules.link_open = function (tokens, idx, options, env, self) {
-			if (isContainerOpen()) {
-				tokens[idx].attrPush(['class', 'md-tip-link']);
-			}
-			
-			return defaultRender(tokens, idx, options, env, self);
-		};
-	}
-	
-	function init() {
-        setupContainer('note');
-        setupContainer('tip');
-        setupContainer('important');
-        setupContainer('caution');
-        setupContainer('warning');
-		
-		if (links) {
-			setupLinks();
-		}
-	}
-};
+        }
+    };
+})();

package/hdoc-build.js

@@ -1,14 +1,30 @@
+const {
+    stat
+} = require('fs');
+
 (function () {
     'use strict';
 
     const dree = require('dree'),
         fs = require('fs-extra'),
+        mdfm = require('markdown-it-front-matter'),
         path = require('path'),
+        Database = require('better-sqlite3'),
         URL = require("url").URL,
         validate = require(path.join(__dirname, 'hdoc-validate.js')),
         hdoc = require(path.join(__dirname, 'hdoc-module.js')),
+        hdoc_index = require(path.join(__dirname, 'hdoc-db.js')),
         zipper = require('zip-local');
-  
+
+    const h_tags_to_search = ['h1', 'h2', 'h3'],
+        index_cols = [
+            'relative_url',
+            'book_id',
+            'book_audience',
+            'book_tags',
+            'doc_title',
+            'doc_content'
+        ];
 
     let conversion_attempted = 0,
         conversion_success = 0,
@@ -16,17 +32,89 @@
         includes_found = 0,
         includes_success = 0,
         includes_failed = 0,
+        hdocbook_project,
         docId = '',
-        md_files = [];
+        md_files = [],
+        static_html_files = [],
+        index_records = [];
+
+    const transform_static_html = function (file_path) {
+        if (fs.existsSync(file_path.path)) {
+            // Load HTML file
+            let html_txt = fs.readFileSync(file_path.path, 'utf8');
+            html_txt = html_txt.replace(/\r/gm, ''); // Remove CR's so we're just dealing with newlines
+
+            let html_txt_updated = false;
+
+            // Check if we have a frontmatter comment
+            const fm_header = hdoc.getHTMLFrontmatterHeader(html_txt);
+            if (Object.keys(fm_header.fm_properties).length > 0) {
+                // We have some frontmatter headers, check if title is one of them
+                let fm_title_found = false;
+                if (fm_header.fm_properties && fm_header.fm_properties.title !== undefined) {
+                    // We have a title - but does the title have a value
+                    if (fm_header.fm_properties.title === '') {
+                        // No value - remove title from the properties map 
+                        // so we don't end up with 2 title properties, one empty and one with a value
+                        delete fm_header.fm_properties.title;
+                    } else {
+                        // We have a value for the title property
+                        fm_title_found = true;
+                    }
+                }
+
+                if (!fm_title_found) {
+                    // No frontmatter title found in properties
+                    // Go get title from h tags in html
+                    const html_heading = hdoc.getFirstHTMLHeading(html_txt, h_tags_to_search);
+
+                    if (html_heading && html_heading[0] && html_heading[0].children && html_heading[0].children[0] && html_heading[0].children[0].data) {
+                        // We've found a heading tag, add that as a title to the existing frontmatter properties
+                        let frontmatter_header = `[[FRONTMATTER\ntitle: ${html_heading[0].children[0].data}`;
+                        for (const key in fm_header.fm_properties) {
+                            if (fm_header.fm_properties.hasOwnProperty(key)) {
+                                frontmatter_header += `\n${key}: ${fm_header.fm_properties[key]}`;
+                            }
+                        }
+                        frontmatter_header += '\n]]';
+                        html_txt = html_txt.replace(fm_header.fm_header, frontmatter_header);
+                        html_txt_updated = true;
+                    } else {
+                        // No header tag, no frontmatter title, output a warning 
+                        console.log(`No frontmatter title property, or ${h_tags_to_search.join(', ')} tags detected in ${file_path.path}`);
+                    }
+                }
+            } else {
+                // We have no frontmatter headers, get and build one from the html headings
+                const html_heading = hdoc.getFirstHTMLHeading(html_txt, h_tags_to_search);
+                if (html_heading && html_heading[0] && html_heading[0].children && html_heading[0].children[0] && html_heading[0].children[0].data) {
+                    // We've found a heading tag, add that as a title to the frontmatter content
+                    const frontmatter_header = `<!--[[FRONTMATTER\r\ntitle: ${html_heading[0].children[0].data}\r\n]]-->\r\n`;
+                    html_txt = frontmatter_header + html_txt;
+                    html_txt_updated = true;
+                } else {
+                    // No header tag, no frontmatter title, output a warning 
+                    console.log(`No frontmatter title property, or ${h_tags_to_search.join(', ')} tags detected in ${file_path.path}`);
+                }
+            }
+            index_records.push({relative_path: file_path.relativePath, index_html: hdoc_index.transform_html_for_index(html_txt)});
+            if (html_txt_updated) {
+                // Save HTML into HTML file
+                fs.writeFile(file_path.path, html_txt, function writeJSON(err) {
+                    if (err) return console.log('Error writing:', target_file, '\r\n', err);
+                });
+            }
+        }
+    };
 
-    const transform_markdown_and_save_html  = function (file_path, md) {
+    const transform_markdown_and_save_html = function (file_path) {
         conversion_attempted++;
-        if (fs.existsSync(file_path)) {
+        if (fs.existsSync(file_path.path)) {
             // Load markdown file
-            let md_txt = hdoc.expand_variables(fs.readFileSync(file_path, 'utf8'));
+            let md_txt = hdoc.expand_variables(fs.readFileSync(file_path.path, 'utf8'));
 
             // Pull in external includes
-            const includes_processed = hdoc.process_includes(file_path, md_txt);
+            const includes_processed = hdoc.process_includes(file_path.path, md_txt);
             md_txt = includes_processed.body;
             includes_found += includes_processed.found;
             includes_success += includes_processed.success;
@@ -37,60 +125,94 @@
                 }
             }
 
-            const mdfm = require('markdown-it-front-matter');
+            // One markdown parser per file. Seems wrong, but doesn't work with a global one once past the first md file
+            // Steve - revisit this
             const md = require('markdown-it')({
-                // enable everything
                 html: true,
                 linkify: true,
-                typographer: true,
-                //  highlight: function (str, lang) {
-                //      if (lang && hljs.getLanguage(lang)) {
-                //        try {
-                //          return hljs.highlight(lang, str, true).value;
-                //        } catch (__) {}
-                //      }
-                //  }
-                });
-        
-            var frontmatter_content = "";
+                typographer: true
+            });
+
+            // Process Frontmatter tags
+            let frontmatter_content = "";
             md.use(mdfm, function (fm) {
                 frontmatter_content = fm;
-            });            
+            });
 
+            // Process tips
             const tips = require(__dirname + '/custom_modules/tips.js');
             md.use(tips, {
-               links: true
+                links: true
             });
 
             // Render markdown into HTML
-            var html_txt = md.render(md_txt.toString());
+            let html_txt = md.render(md_txt.toString());
+
+            // Does frontmatter tag contain a title property
+            let fm_contains_title = false;
+            let fm_content = frontmatter_content.split(/\r?\n/);
+            if (fm_content.length >= 0) {
+                fm_content.forEach(function (fm_prop) {
+                    const fm_property = fm_prop.split(':');
+                    if (fm_property[0] && fm_property[0] === 'title' && fm_property[1] && fm_property[1].length > 0) fm_contains_title = true;
+                });
+            }
 
+            if (!fm_contains_title) {
+                // Frontmatter tags don't contain a title property - go pull the first one from the html heading tags
+                const html_heading = hdoc.getFirstHTMLHeading(html_txt, h_tags_to_search);
+
+                if (html_heading && html_heading[0] && html_heading[0].children && html_heading[0].children[0] && html_heading[0].children[0].data) {
+                    // We've found a heading tag, add that as a title to the frontmatter content
+                    if (frontmatter_content.length > 0) frontmatter_content += '\r\n';
+                    frontmatter_content += `title: ${html_heading[0].children[0].data}`;
+                } else {
+                    // No header tag, no frontmatter title, output a warning 
+                    console.log(`No frontmatter title property, or h1, h2 or h3 header tags detected in ${file_path}`);
+                }
+            }
+
+            // Add frontmatter tags as comment to front of HTML 
             if (frontmatter_content.length) {
                 html_txt = "<!--[[FRONTMATTER\r\n" + frontmatter_content + "\r\n]]-->\r\n" + html_txt;
             }
- 
+
             // Save HTML into HTML file
-            const target_file = file_path.replace(path.extname(file_path), '.html');
+            const target_file = file_path.path.replace(path.extname(file_path.path), '.html');
+            const relative_path = file_path.relativePath.replace(path.extname(file_path.path), '.html');
             fs.writeFile(target_file, html_txt, function writeJSON(err) {
                 if (err) return console.log('Error writing:', target_file, '\r\n', err);
             });
+
+            index_records.push({relative_path: relative_path, index_html: hdoc_index.transform_html_for_index(html_txt)});
+
+            // Delete MD file from _work path
+            try {
+                fs.unlinkSync(file_path.path);
+            } catch (e) {
+                console.log(`Error deleting ${file_path.path}: ${e}`);
+            }
             conversion_success++;
             return true;
         }
         conversion_failed++;
-        console.error('MD file does not exist:', file_path);
+        console.error('MD file does not exist:', file_path.path);
         return false;
     };
 
     // File callbacks for scans
     const fileCallback = function (element) {
-        md_files.push(element.path);
+        if (element.extension === 'md') {
+            md_files.push(element);
+        } else {
+            static_html_files.push(element);
+        }
     };
 
     const dreeOptions = {
         descendants: true,
         depth: 10,
-        extensions: ['md'],
+        extensions: ['md', 'html', 'htm'],
         hash: false,
         normalize: true,
         size: false,
@@ -99,13 +221,13 @@
         symbolicLinks: false
     };
 
-    exports.run = function (source_path, md, verbose) {
+    exports.run = function (source_path, verbose) {
         // GERRY: The purpose of this function is to create a zip file containing the hdocbook content, 
         // * Create a _work folder 
         // * copy the hdocbook content to the work folder
         // * Render all markdown into side-by-side HTML file
         // * Replace SERVER_VARS embedded in documents with the right version information etc.
-        // * Build an index (sqlite FTS5) by extracting text from all HTML content in  the work 
+        // * Build an index (sqlite FTS5) by extracting text from all HTML content in the work 
         //   folder, conceptually we are making a little mini website crawler to index all of the content
         //   within the book. 
         // * Package everything up into a ZIP file, ready for the build controller to package and publish
@@ -115,8 +237,8 @@
 
         // Load the hdocbook-project.json file to get the docId
         // use the docId to get the book config
-        const hdocbook_project_config_path = path.join(source_path, 'hdocbook-project.json'),
-            hdocbook_project = require(hdocbook_project_config_path);
+        const hdocbook_project_config_path = path.join(source_path, 'hdocbook-project.json');
+        hdocbook_project = require(hdocbook_project_config_path);
 
         docId = hdocbook_project.docId;
 
@@ -150,8 +272,15 @@
 
         // Work through MD files and convert to HTML
         md_files.forEach(function (md_file) {
-            transform_markdown_and_save_html(md_file, md);
+            transform_markdown_and_save_html(md_file);
+        });
+
+        // Work through Static HTML files and add Frontmatter tags
+        static_html_files.forEach(function (static_html_file) {
+            transform_static_html(static_html_file);
         });
+
+
         console.log(`                MD files found: ${conversion_attempted}`);
         console.log(`Successfully converted to HTML: ${conversion_success}`);
         console.log(`             Failed to convert: ${conversion_failed}\r\n`);
@@ -159,15 +288,50 @@
         console.log(`              Includes Success: ${includes_success}`);
         console.log(`               Includes Failed: ${includes_failed}\r\n`);
 
+        console.log(`       Static HTML Files Found: ${static_html_files.length}\r\n`);
+
+        // Validate content
         const validation_success = validate.run(work_path, docId, verbose);
         if (!validation_success) {
             process.exit(1);
         }
 
-        const zip_path = path.join(work_path, docId + '.zip');
-        zipper.sync.zip(path.join(work_path, docId)).compress().save(zip_path);
+        // Now build the index
+        console.log('Performing SQlite index creation...\n');
+        let db_name = path.join(work_path, docId, docId + '.db');
+        const db = new Database(db_name);
+        console.log(`Database created: ${db_name}`);
+
+        // Now add the table
+        const table_name = 'hdoc_index';
+        const table_created = hdoc_index.create_virtual_table(db, table_name, index_cols);
+
+        if (table_created !== null) {
+            console.log(`Error creating table: ${table_created}`);
+        } else {
+            console.log(`Virtual table created: ${table_created}`);
+            if (!hdocbook_config.tags) hdocbook_config.tags = [];
+            for (let i = 0; i < index_records.length; i++) {
+                const index_vals = [
+                    index_records[i].relative_path.replace('\\', '/').replace(`${hdocbook_config.docId}/`, ''),
+                    hdocbook_config.docId,
+                    hdocbook_config.audience.join(','),
+                    hdocbook_config.tags.join(','),
+                    index_records[i].index_html.fm_props.title,
+                    index_records[i].index_html.text
+                ];
+                hdoc_index.insert_record(db, table_name, index_cols, index_vals, hdocbook_config.docId, index_records[i].index_html.fm_props.title);
+            }
+        }
+
+        try {
+            const zip_path = path.join(work_path, docId + '.zip');
+            zipper.sync.zip(path.join(work_path, docId)).compress().save(zip_path);
+            console.log(`\nZIP Creation Success: ${zip_path}\r\n`);
+            console.log('Build Complete\r\n');
+        } catch (e) {
+            console.log('\nError creating ZIP: ' + e);
+        }
 
-        console.log(`ZIP Creation Success: ${zip_path}\r\n`);
-        console.log('Build Complete\r\n');
     };
 })();

package/hdoc-db.js

@@ -1,158 +1,71 @@
-const { nextTick } = require('process');
-
-(function() {
+(function () {
     'use strict';
 
-    const   dree = require('dree'),
-            fs = require('fs-extra'),
-            html2text = require('html-to-text'),
-            sqlite3 = require('sqlite3'),
-            path = require('path');
-
-    const  table_name = 'hbdocs';
-
-    let     db_name = 'hbdocs.db',
-            hdocbook_meta = null,
-            md_files = [];
+    const html2text = require('html-to-text'),
+        path = require('path'),
+        hdoc = require(path.join(__dirname, 'hdoc-module.js'));
 
-    // File callbacks for scan
-    const file_callback = function (element) {
-        if (path.extname(element.path) === '.md' && !element.path.includes('_work')) {
-            md_files.push(element);
-        } else if (element.name === 'hdocbook.json') {
-            // hdocbook meta data, read and store
-            hdocbook_meta = require(element.path);
+    exports.create_virtual_table = function (db, table_name, columns) {
+        let create_sql = [];
+        create_sql.push(`create VIRTUAL table ${table_name} USING fts5(`);
+        for (let i = 0; i < columns.length; i++) {
+            if (i !== 0) create_sql.push(`,${columns[i]}`);
+            else create_sql.push(columns[i]);
+        }
+        create_sql.push(');');
+        try {
+            db.exec(create_sql.join('\n'));
+            return null;
+        } catch (e) {
+            return e;
         }
     };
 
-    const create_table = function(db) {
-        db.exec(`
-        create table ${table_name} (
-        relative_url text primary key not null,
-        book_id text not null,
-        title text not null,
-        content text not null,
-        tags text null,
-        audience text not null
-        );
-        `, (err)  => {
-            if (err !== null) {
-                console.error(`Error creating table [${table_name}]: ${err}`);
-                return false;
+    exports.insert_record = function (db, table, columns, values, book_id, doc_title) {
+        let queryProps = [];
+        queryProps.push(`INSERT INTO ${table}`);
+        let cols = '(';
+        let vals = 'VALUES (';
+        for (let i = 0; i < columns.length; i++) {
+            if (i === 0) {
+                cols += `${columns[i]}`;
+                vals += '?';
             } else {
-                console.log(`  Table created successfully: ${table_name}`);
-                return true;
+                cols += `,${columns[i]}`;
+                vals += ',?';
             }
-        });
-        return true;
-    };
-
-    const build_db = function(db, md, book_def, md_files) {
-        for (let i = 0; i < md_files.length; i++) {
-            db.exec(`
-            insert into ${table_name} 
-                (book_id, title, relative_url, content, tags, audience)
-            values
-                (
-                    '${book_def.docId}',
-                    '${book_def.title}',
-                    '${md_files[i].relativePath}',
-                    'Some file content',
-                    '${book_def.tags.join(';')}',
-                    '${book_def.audience.join(';')}'
-                );
-            `, (err)  => {
-                if (err !== null) {
-                    console.error(`Error inserting record [${md_files[i].relativePath}]: ${err}`);
-                    return false;
-                } else {
-                    console.log(`  Record inserted successfully: ${md_files[i].relativePath}`);
-                    return true;
-                }
-            });
+        }
+        cols += ')';
+        vals += ')';
+        queryProps.push(cols);
+        queryProps.push(vals);
+
+        try {
+            const stmt = db.prepare(queryProps.join('\n'));
+            const info = stmt.run(values);
+            console.log(`Inserted index record ${info.lastInsertRowid}: ${book_id} - ${doc_title}`);
+        } catch (e) {
+            console.log(`Index record creation failed - ${book_id}/${doc_title}: ${e}`);
         }
     };
 
-    const transform_markdown  = function (file_path, md) {
-        if (fs.existsSync(file_path)) {
-            // Load markdown file
-            let md_txt = hdoc.expand_variables(fs.readFileSync(file_path, 'utf8'));
-
-            // Pull in external includes
-            const includes_processed = hdoc.process_includes(file_path, md_txt);
-            md_txt = includes_processed.body;
-            includes_found += includes_processed.found;
-            includes_success += includes_processed.success;
-            includes_failed += includes_processed.failed;
-            if (includes_processed.errors.length > 0) {
-                for (let i = 0; i < includes_processed.errors.length; i++) {
-                    console.error(includes_processed.errors[i]);
-                }
-            }
+    exports.transform_html_for_index = function (html_txt) {
+        let response = {
+            text: '',
+            fm_props: {}
+        };
 
-            // Render markdown into HTML
-            var html_txt = md.render(md_txt.toString());
+        // Get frontmatter properties
+        const fm_headers = hdoc.getHTMLFrontmatterHeader(html_txt);
+        response.fm_props = fm_headers.fm_properties;
 
-            // Save HTML into HTML file
-            const target_file = file_path.replace(path.extname(file_path), '.html');
-            fs.writeFile(target_file, html_txt, function writeJSON(err) {
-                if (err) return console.log('Error writing:', target_file, '\r\n', err);
-            });
-            conversion_success++;
-            return true;
-        }
-        conversion_failed++;
-        console.error('MD file does not exist:', file_path);
-        return false;
-    };
+        // Convert HTML into plain text
+        response.text = html2text.convert(html_txt, {
+            wordwrap: null
+        });
 
-    const dree_options = {
-        descendants: true,
-        depth: 10,
-        extensions: ['md','json'],
-        hash: false,
-        normalize: true,
-        size: false,
-        sizeInBytes: false,
-        stat: false,
-        symbolicLinks: false
+        return response;
     };
 
-    exports.run = function(source_path, md) {
-        /*
-            STEVE - The purpose of this command is to allow content developers to do local builds of
-                    the SQlite database and index file, for validation before commit and build 
-        */
-        dree.scan(source_path, dree_options, file_callback);
-
-        if (hdocbook_meta === null) {
-            console.error('hdocbook.json was not found, or has no content');
-            process.exit(1);
-        }
-        if (md_files.length === 0) {
-            console.error('No markdown files detected in', source_path);
-            process.exit(1);
-        }
 
-        db_name = path.join(source_path, db_name);
-        if (fs.existsSync(db_name)) {
-            try {
-                fs.removeSync(db_name);
-            } catch (e) {
-                console.error(`Failed to delete existing db file: ${db_name}`);
-            }
-        }
-
-        let db = new sqlite3.Database(db_name, (err) => {
-            if (err) {
-                console.error('Error creating database:', err);
-                process.exit(1);
-            }
-            console.log(`DB file created successfully: ${db_name}`);
-            if (create_table(db)) {
-                if (!hdocbook_meta.tags) hdocbook_meta.tags = [];
-                build_db(db, md, hdocbook_meta, md_files );
-            }
-        });
-    };
 })();

package/hdoc-module.js

@@ -1,7 +1,8 @@
 (function () {
     'use strict';
 
-    const request = require('sync-request');
+    const   cheerio = require('cheerio'),
+            request = require('sync-request');
 
     let includesCache = {};
 
@@ -143,4 +144,45 @@
         response.body = body;
         return response;
     };
+
+    // Takes html, returns the first heading detected in the order provided in h_to_search
+    // Looks for h1 tags first, then hX, hY, hZ in order
+    exports.getFirstHTMLHeading = function (html_body, h_to_search = ['h1']) {
+        const $ = cheerio.load(html_body);
+        for (let i = 0; i < h_to_search.length; i++) {
+            let heading = $(h_to_search[i]).map(function (i) {
+                return $(this);
+            }).get();
+            if (heading.length > 0) {
+                return heading[0];
+            } 
+        }
+        return false;
+    };
+
+    exports.getHTMLFrontmatterHeader = function(html_body) {
+        let response = {
+            fm_header: '',
+            fm_properties: {}
+        };
+        const $ = cheerio.load(html_body, { decodeEntities: false });
+        if ($._root && $._root.children && $._root.children instanceof Array && $._root.children.length > 0) {
+            $._root.children.forEach(function(child){
+                if (child.type === 'comment' && child.data && child.data.startsWith('[[FRONTMATTER')) {
+                    // We have a Frontmatter header - return each property in an array
+                    const fm_properties = child.data.split(/\r?\n/);
+                    for (let i = 0; i < fm_properties.length; i++) {
+                        const property_details = fm_properties[i].split(':', 2);
+                        if (property_details.length > 1) {
+                            response.fm_properties[property_details[0].trim().toLowerCase()] = property_details[1].trim();
+                        }
+                    }
+
+                    // And return the header as a whole so it can be easily replaced
+                    response.fm_header = child.data;
+                }
+            });
+        }
+        return response;
+    };
 })();

package/hdoc-serve.js

@@ -92,7 +92,6 @@
                }
             }
 
-            const mdfm = require('markdown-it-front-matter');
             const md = require('markdown-it')({
                 // enable everything
                 html: true,

package/hdoc.js

@@ -1,37 +1,14 @@
 #!/usr/bin/env node
 
-const { createCipheriv } = require('crypto');
-
 (function () {
    'use strict';
 
-   const preRun = require('./validateNodeVer.js');
-   const fs = require('fs');
-   const path = require('path');
-   const stream = require('stream');
-   const yaml = require('js-yaml');
-   const mdfm = require('markdown-it-front-matter');
-   const hljs = require('highlight.js'); // https://highlightjs.org/
-   const md = require('markdown-it')({
-      // enable everything
-      html: true,
-      linkify: true,
-      typographer: true,
-      //  highlight: function (str, lang) {
-      //      if (lang && hljs.getLanguage(lang)) {
-      //        try {
-      //          return hljs.highlight(lang, str, true).value;
-      //        } catch (__) {}
-      //      }
-      //  }
-   });
-
-   var frontmatter_content;
-   md.use(mdfm, function (fm) {
-      frontmatter_content = fm;
-   });
+   const preRun = require('./validateNodeVer.js'),
+      fs = require('fs'),
+      path = require('path');
 
    const packageFile = path.join(__dirname, 'package.json');
+
    const getHdocPackageVersion = function (packagePath) {
       if (fs.existsSync(packagePath)) {
          try {
@@ -44,19 +21,13 @@ const { createCipheriv } = require('crypto');
          console.error('Package file not found: ', packagePath);
       }
    };
-   const tips = require(__dirname + '/custom_modules/tips.js');
-
-   md.use(tips, {
-      links: true
-   });
 
    // Default source path to working directory
    let source_path = process.cwd();
    let ui_path = path.join(__dirname, 'ui');
 
-   let   command = '', // Our command to run
-         verbose = false,
-         build_index = false;
+   let command = '', // Our command to run
+      verbose = false;
 
    // Get options from command args
    for (let x = 0; x < process.argv.length; x++) {
@@ -93,22 +64,19 @@ const { createCipheriv } = require('crypto');
 
    if (command == 'serve') {
       const server = require(path.join(__dirname, 'hdoc-serve.js'));
-      server.run(ui_path, source_path, md);
+      server.run(ui_path, source_path);
    } else if (command == 'build') {
       const builder = require(path.join(__dirname, 'hdoc-build.js'));
-      builder.run(source_path, md, verbose);
+      builder.run(source_path, verbose);
    } else if (command == 'stats') {
       const stats = require(path.join(__dirname, 'hdoc-stats.js'));
-      stats.run(ui_path, source_path, md, verbose);
+      stats.run(ui_path, source_path, verbose);
    } else if (command == 'init') {
       const init = require(path.join(__dirname, 'hdoc-init.js'));
-      init.run(__dirname, source_path, md);
+      init.run(__dirname, source_path);
    } else if (command == 'help') {
       const help = require(path.join(__dirname, 'hdoc-help.js'));
       help.run();
-   } else if (command == 'buildindex') {
-      const build_db = require(path.join(__dirname, 'hdoc-db.js'));
-      build_db.run(source_path, md);
    } else {
       console.log('Unknown command:', command, '\r\n');
       console.log('Run hdoc help for information regarding this tool.\r\n');

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "hdoc-tools",
-  "version": "0.7.26",
+  "version": "0.8.0",
   "description": "Hornbill HDocBook Development Support Tool",
   "main": "hdoc.js",
   "bin": {
@@ -28,6 +28,7 @@
   "author": "Hornbill Technologies Ltd",
   "license": "ISC",
   "dependencies": {
+    "better-sqlite3": "^8.0.1",
     "body-parser": "^1.20.1",
     "cheerio": "^1.0.0-rc.12",
     "cookie-parser": "^1.4.6",
@@ -42,7 +43,6 @@
     "markdown-it-front-matter": "^0.2.3",
     "multer": "^1.4.5-lts.1",
     "prompt": "^1.3.0",
-    "sqlite3": "^5.1.4",
     "stream": "0.0.2",
     "sync-request": "^6.1.0",
     "words-count": "^2.0.2",