hdoc-tools 0.7.27 → 0.8.0

package/hdoc-build.js

@@ -1,4 +1,6 @@
-const { stat } = require('fs');
+const {
+    stat
+} = require('fs');
 
 (function () {
     'use strict';
@@ -7,12 +9,22 @@ const { stat } = require('fs');
         fs = require('fs-extra'),
         mdfm = require('markdown-it-front-matter'),
         path = require('path'),
+        Database = require('better-sqlite3'),
         URL = require("url").URL,
         validate = require(path.join(__dirname, 'hdoc-validate.js')),
         hdoc = require(path.join(__dirname, 'hdoc-module.js')),
+        hdoc_index = require(path.join(__dirname, 'hdoc-db.js')),
         zipper = require('zip-local');
 
-    const h_tags_to_search = ['h1', 'h2', 'h3'];
+    const h_tags_to_search = ['h1', 'h2', 'h3'],
+        index_cols = [
+            'relative_url',
+            'book_id',
+            'book_audience',
+            'book_tags',
+            'doc_title',
+            'doc_content'
+        ];
 
     let conversion_attempted = 0,
         conversion_success = 0,
@@ -20,55 +32,56 @@ const { stat } = require('fs');
         includes_found = 0,
         includes_success = 0,
         includes_failed = 0,
+        hdocbook_project,
         docId = '',
         md_files = [],
-        static_html_files = [];
+        static_html_files = [],
+        index_records = [];
 
-    const transform_html = function(file_path) {
-        if (fs.existsSync(file_path)) {
+    const transform_static_html = function (file_path) {
+        if (fs.existsSync(file_path.path)) {
             // Load HTML file
-            let html_txt = fs.readFileSync(file_path, 'utf8');
-            html_txt = html_txt.replace(/\r/gm,''); // Remove CR's so we're just dealing with newlines
+            let html_txt = fs.readFileSync(file_path.path, 'utf8');
+            html_txt = html_txt.replace(/\r/gm, ''); // Remove CR's so we're just dealing with newlines
 
             let html_txt_updated = false;
-            
+
             // Check if we have a frontmatter comment
             const fm_header = hdoc.getHTMLFrontmatterHeader(html_txt);
-            if (fm_header.fm_properties.length > 0 ) {
+            if (Object.keys(fm_header.fm_properties).length > 0) {
                 // We have some frontmatter headers, check if title is one of them
                 let fm_title_found = false;
-                for (let i = 0; i < fm_header.fm_properties.length; i++) {
-                    if (fm_header.fm_properties[i].key.toLowerCase() === 'title') {
-
-                        // We have a title - bur does the title have a value
-                        if (fm_header.fm_properties[i].value === '') {
-                            // No value - remove title from the properties array 
-                            // so we don't end up with 2 title properties, one empty and one with a value
-                            fm_header.fm_properties.splice(i, 1);
-                        } else {
-                            // We have a value for the title property
-                            fm_title_found = true;
-                        }
+                if (fm_header.fm_properties && fm_header.fm_properties.title !== undefined) {
+                    // We have a title - but does the title have a value
+                    if (fm_header.fm_properties.title === '') {
+                        // No value - remove title from the properties map 
+                        // so we don't end up with 2 title properties, one empty and one with a value
+                        delete fm_header.fm_properties.title;
+                    } else {
+                        // We have a value for the title property
+                        fm_title_found = true;
                     }
                 }
-                
+
                 if (!fm_title_found) {
                     // No frontmatter title found in properties
                     // Go get title from h tags in html
                     const html_heading = hdoc.getFirstHTMLHeading(html_txt, h_tags_to_search);
-    
+
                     if (html_heading && html_heading[0] && html_heading[0].children && html_heading[0].children[0] && html_heading[0].children[0].data) {
                         // We've found a heading tag, add that as a title to the existing frontmatter properties
                         let frontmatter_header = `[[FRONTMATTER\ntitle: ${html_heading[0].children[0].data}`;
-                        fm_header.fm_properties.forEach(function(fm_prop){
-                            frontmatter_header += `\n${fm_prop.key}: ${fm_prop.value}`;
-                        });
+                        for (const key in fm_header.fm_properties) {
+                            if (fm_header.fm_properties.hasOwnProperty(key)) {
+                                frontmatter_header += `\n${key}: ${fm_header.fm_properties[key]}`;
+                            }
+                        }
                         frontmatter_header += '\n]]';
                         html_txt = html_txt.replace(fm_header.fm_header, frontmatter_header);
                         html_txt_updated = true;
                     } else {
                         // No header tag, no frontmatter title, output a warning 
-                        console.log(`No frontmatter title property, or ${h_tags_to_search.join(', ')} tags detected in ${file_path}`);
+                        console.log(`No frontmatter title property, or ${h_tags_to_search.join(', ')} tags detected in ${file_path.path}`);
                     }
                 }
             } else {
@@ -81,26 +94,27 @@ const { stat } = require('fs');
                     html_txt_updated = true;
                 } else {
                     // No header tag, no frontmatter title, output a warning 
-                    console.log(`No frontmatter title property, or ${h_tags_to_search.join(', ')} tags detected in ${file_path}`);
+                    console.log(`No frontmatter title property, or ${h_tags_to_search.join(', ')} tags detected in ${file_path.path}`);
                 }
             }
+            index_records.push({relative_path: file_path.relativePath, index_html: hdoc_index.transform_html_for_index(html_txt)});
             if (html_txt_updated) {
                 // Save HTML into HTML file
-                fs.writeFile(file_path, html_txt, function writeJSON(err) {
+                fs.writeFile(file_path.path, html_txt, function writeJSON(err) {
                     if (err) return console.log('Error writing:', target_file, '\r\n', err);
                 });
             }
         }
     };
 
-    const transform_markdown_and_save_html  = function (file_path) {
+    const transform_markdown_and_save_html = function (file_path) {
         conversion_attempted++;
-        if (fs.existsSync(file_path)) {
+        if (fs.existsSync(file_path.path)) {
             // Load markdown file
-            let md_txt = hdoc.expand_variables(fs.readFileSync(file_path, 'utf8'));
+            let md_txt = hdoc.expand_variables(fs.readFileSync(file_path.path, 'utf8'));
 
             // Pull in external includes
-            const includes_processed = hdoc.process_includes(file_path, md_txt);
+            const includes_processed = hdoc.process_includes(file_path.path, md_txt);
             md_txt = includes_processed.body;
             includes_found += includes_processed.found;
             includes_success += includes_processed.success;
@@ -123,12 +137,12 @@ const { stat } = require('fs');
             let frontmatter_content = "";
             md.use(mdfm, function (fm) {
                 frontmatter_content = fm;
-            });            
+            });
 
             // Process tips
             const tips = require(__dirname + '/custom_modules/tips.js');
             md.use(tips, {
-               links: true
+                links: true
             });
 
             // Render markdown into HTML
@@ -138,7 +152,7 @@ const { stat } = require('fs');
             let fm_contains_title = false;
             let fm_content = frontmatter_content.split(/\r?\n/);
             if (fm_content.length >= 0) {
-                fm_content.forEach(function(fm_prop){
+                fm_content.forEach(function (fm_prop) {
                     const fm_property = fm_prop.split(':');
                     if (fm_property[0] && fm_property[0] === 'title' && fm_property[1] && fm_property[1].length > 0) fm_contains_title = true;
                 });
@@ -162,40 +176,43 @@ const { stat } = require('fs');
             if (frontmatter_content.length) {
                 html_txt = "<!--[[FRONTMATTER\r\n" + frontmatter_content + "\r\n]]-->\r\n" + html_txt;
             }
- 
+
             // Save HTML into HTML file
-            const target_file = file_path.replace(path.extname(file_path), '.html');
+            const target_file = file_path.path.replace(path.extname(file_path.path), '.html');
+            const relative_path = file_path.relativePath.replace(path.extname(file_path.path), '.html');
             fs.writeFile(target_file, html_txt, function writeJSON(err) {
                 if (err) return console.log('Error writing:', target_file, '\r\n', err);
             });
 
+            index_records.push({relative_path: relative_path, index_html: hdoc_index.transform_html_for_index(html_txt)});
+
             // Delete MD file from _work path
             try {
-                fs.unlinkSync(file_path);
+                fs.unlinkSync(file_path.path);
             } catch (e) {
-                console.log(`Error deleting ${file_path}: ${e}`);
+                console.log(`Error deleting ${file_path.path}: ${e}`);
             }
             conversion_success++;
             return true;
         }
         conversion_failed++;
-        console.error('MD file does not exist:', file_path);
+        console.error('MD file does not exist:', file_path.path);
         return false;
     };
 
     // File callbacks for scans
     const fileCallback = function (element) {
         if (element.extension === 'md') {
-            md_files.push(element.path);
+            md_files.push(element);
         } else {
-            static_html_files.push(element.path);
+            static_html_files.push(element);
         }
     };
 
     const dreeOptions = {
         descendants: true,
         depth: 10,
-        extensions: ['md','html','htm'],
+        extensions: ['md', 'html', 'htm'],
         hash: false,
         normalize: true,
         size: false,
@@ -210,7 +227,7 @@ const { stat } = require('fs');
         // * copy the hdocbook content to the work folder
         // * Render all markdown into side-by-side HTML file
         // * Replace SERVER_VARS embedded in documents with the right version information etc.
-        // * Build an index (sqlite FTS5) by extracting text from all HTML content in  the work 
+        // * Build an index (sqlite FTS5) by extracting text from all HTML content in the work 
         //   folder, conceptually we are making a little mini website crawler to index all of the content
         //   within the book. 
         // * Package everything up into a ZIP file, ready for the build controller to package and publish
@@ -220,8 +237,8 @@ const { stat } = require('fs');
 
         // Load the hdocbook-project.json file to get the docId
         // use the docId to get the book config
-        const hdocbook_project_config_path = path.join(source_path, 'hdocbook-project.json'),
-            hdocbook_project = require(hdocbook_project_config_path);
+        const hdocbook_project_config_path = path.join(source_path, 'hdocbook-project.json');
+        hdocbook_project = require(hdocbook_project_config_path);
 
         docId = hdocbook_project.docId;
 
@@ -259,10 +276,11 @@ const { stat } = require('fs');
         });
 
         // Work through Static HTML files and add Frontmatter tags
-        static_html_files.forEach(function(static_html_file){
-            transform_html(static_html_file);
+        static_html_files.forEach(function (static_html_file) {
+            transform_static_html(static_html_file);
         });
 
+
         console.log(`                MD files found: ${conversion_attempted}`);
         console.log(`Successfully converted to HTML: ${conversion_success}`);
         console.log(`             Failed to convert: ${conversion_failed}\r\n`);
@@ -272,15 +290,48 @@ const { stat } = require('fs');
 
         console.log(`       Static HTML Files Found: ${static_html_files.length}\r\n`);
 
+        // Validate content
         const validation_success = validate.run(work_path, docId, verbose);
         if (!validation_success) {
             process.exit(1);
         }
 
-        const zip_path = path.join(work_path, docId + '.zip');
-        zipper.sync.zip(path.join(work_path, docId)).compress().save(zip_path);
+        // Now build the index
+        console.log('Performing SQlite index creation...\n');
+        let db_name = path.join(work_path, docId, docId + '.db');
+        const db = new Database(db_name);
+        console.log(`Database created: ${db_name}`);
+
+        // Now add the table
+        const table_name = 'hdoc_index';
+        const table_created = hdoc_index.create_virtual_table(db, table_name, index_cols);
+
+        if (table_created !== null) {
+            console.log(`Error creating table: ${table_created}`);
+        } else {
+            console.log(`Virtual table created: ${table_created}`);
+            if (!hdocbook_config.tags) hdocbook_config.tags = [];
+            for (let i = 0; i < index_records.length; i++) {
+                const index_vals = [
+                    index_records[i].relative_path.replace('\\', '/').replace(`${hdocbook_config.docId}/`, ''),
+                    hdocbook_config.docId,
+                    hdocbook_config.audience.join(','),
+                    hdocbook_config.tags.join(','),
+                    index_records[i].index_html.fm_props.title,
+                    index_records[i].index_html.text
+                ];
+                hdoc_index.insert_record(db, table_name, index_cols, index_vals, hdocbook_config.docId, index_records[i].index_html.fm_props.title);
+            }
+        }
+
+        try {
+            const zip_path = path.join(work_path, docId + '.zip');
+            zipper.sync.zip(path.join(work_path, docId)).compress().save(zip_path);
+            console.log(`\nZIP Creation Success: ${zip_path}\r\n`);
+            console.log('Build Complete\r\n');
+        } catch (e) {
+            console.log('\nError creating ZIP: ' + e);
+        }
 
-        console.log(`ZIP Creation Success: ${zip_path}\r\n`);
-        console.log('Build Complete\r\n');
     };
 })();

package/hdoc-db.js

@@ -1,148 +1,71 @@
-const {
-    nextTick
-} = require('process');
-
 (function () {
     'use strict';
 
-    const dree = require('dree'),
-        fs = require('fs-extra'),
-        html2text = require('html-to-text'),
-        sqlite3 = require('sqlite3'),
+    const html2text = require('html-to-text'),
         path = require('path'),
         hdoc = require(path.join(__dirname, 'hdoc-module.js'));
 
-    const table_name = 'hbdocs';
-
-    let db_name = 'hbdocs.db',
-        hdocbook_meta = null,
-        md_files = [];
-
-    // File callbacks for scan
-    const file_callback = function (element) {
-        if (path.extname(element.path) === '.md' && !element.path.includes('_work')) {
-            md_files.push(element);
-        } else if (element.name === 'hdocbook.json') {
-            // hdocbook meta data, read and store
-            hdocbook_meta = require(element.path);
+    exports.create_virtual_table = function (db, table_name, columns) {
+        let create_sql = [];
+        create_sql.push(`create VIRTUAL table ${table_name} USING fts5(`);
+        for (let i = 0; i < columns.length; i++) {
+            if (i !== 0) create_sql.push(`,${columns[i]}`);
+            else create_sql.push(columns[i]);
+        }
+        create_sql.push(');');
+        try {
+            db.exec(create_sql.join('\n'));
+            return null;
+        } catch (e) {
+            return e;
         }
     };
 
-    const create_table = function (db) {
-        db.exec(`
-        create table ${table_name} (
-        relative_url text primary key not null,
-        book_id text not null,
-        title text not null,
-        content text not null,
-        tags text null,
-        audience text not null
-        );
-        `, (err) => {
-            if (err !== null) {
-                console.error(`Error creating table [${table_name}]: ${err}`);
-                return false;
+    exports.insert_record = function (db, table, columns, values, book_id, doc_title) {
+        let queryProps = [];
+        queryProps.push(`INSERT INTO ${table}`);
+        let cols = '(';
+        let vals = 'VALUES (';
+        for (let i = 0; i < columns.length; i++) {
+            if (i === 0) {
+                cols += `${columns[i]}`;
+                vals += '?';
             } else {
-                console.log(`  Table created successfully: ${table_name}`);
-                return true;
+                cols += `,${columns[i]}`;
+                vals += ',?';
             }
-        });
-        return true;
-    };
-
-    const build_db = function (db, md, book_def, md_files) {
-        for (let i = 0; i < md_files.length; i++) {
-
-            const content = transform_markdown(md_files[i].path, md);
-            db.exec(`
-            insert into ${table_name} 
-                (book_id, title, relative_url, content, tags, audience)
-            values
-                (
-                    '${book_def.docId}',
-                    '${book_def.title}',
-                    '${md_files[i].relativePath}',
-                    '${content}',
-                    '${book_def.tags.join(';')}',
-                    '${book_def.audience.join(';')}'
-                );
-            `, (err) => {
-                if (err !== null) {
-                    console.error(`        Error inserting record: [${md_files[i].relativePath}] ${err}`);
-                    return false;
-                } else {
-                    console.log(`  Record inserted successfully: ${md_files[i].relativePath}`);
-                    return true;
-                }
-            });
+        }
+        cols += ')';
+        vals += ')';
+        queryProps.push(cols);
+        queryProps.push(vals);
+
+        try {
+            const stmt = db.prepare(queryProps.join('\n'));
+            const info = stmt.run(values);
+            console.log(`Inserted index record ${info.lastInsertRowid}: ${book_id} - ${doc_title}`);
+        } catch (e) {
+            console.log(`Index record creation failed - ${book_id}/${doc_title}: ${e}`);
         }
     };
 
-    const transform_markdown = function (file_path, md) {
-        // Load markdown file
-        let md_txt = hdoc.expand_variables(fs.readFileSync(file_path, 'utf8'));
+    exports.transform_html_for_index = function (html_txt) {
+        let response = {
+            text: '',
+            fm_props: {}
+        };
 
-        // Pull in external includes
-        const includes_processed = hdoc.process_includes(file_path, md_txt);
-        md_txt = includes_processed.body;
-
-        // Render markdown into HTML
-        var html_txt = md.render(md_txt.toString());
+        // Get frontmatter properties
+        const fm_headers = hdoc.getHTMLFrontmatterHeader(html_txt);
+        response.fm_props = fm_headers.fm_properties;
 
         // Convert HTML into plain text
-        const text = html2text.convert(html_txt, {
+        response.text = html2text.convert(html_txt, {
             wordwrap: null
-        }); 
-        return text;
-    };
+        });
 
-    const dree_options = {
-        descendants: true,
-        depth: 10,
-        extensions: ['md', 'json'],
-        hash: false,
-        normalize: true,
-        size: false,
-        sizeInBytes: false,
-        stat: false,
-        symbolicLinks: false
+        return response;
     };
 
-    exports.run = function (source_path, md) {
-        /*
-            STEVE - The purpose of this command is to allow content developers to do local builds of
-                    the SQlite database and index file, for validation before commit and build 
-        */
-        dree.scan(source_path, dree_options, file_callback);
-
-        if (hdocbook_meta === null) {
-            console.error('hdocbook.json was not found, or has no content');
-            process.exit(1);
-        }
-        if (md_files.length === 0) {
-            console.error('No markdown files detected in', source_path);
-            process.exit(1);
-        }
 
-        db_name = path.join(source_path, db_name);
-        if (fs.existsSync(db_name)) {
-            try {
-                fs.removeSync(db_name);
-            } catch (e) {
-                console.error(`Failed to delete existing db file: ${db_name}`);
-            }
-        }
-
-        let db = new sqlite3.Database(db_name, (err) => {
-            if (err) {
-                console.error('Error creating database:', err);
-                process.exit(1);
-            }
-            console.log(`DB file created successfully: ${db_name}`);
-            if (create_table(db)) {
-                if (!hdocbook_meta.tags) hdocbook_meta.tags = [];
-                build_db(db, md, hdocbook_meta, md_files);
-            }
-        });
-    };
 })();

package/hdoc-module.js

@@ -163,7 +163,7 @@
     exports.getHTMLFrontmatterHeader = function(html_body) {
         let response = {
             fm_header: '',
-            fm_properties: []
+            fm_properties: {}
         };
         const $ = cheerio.load(html_body, { decodeEntities: false });
         if ($._root && $._root.children && $._root.children instanceof Array && $._root.children.length > 0) {
@@ -174,7 +174,7 @@
                     for (let i = 0; i < fm_properties.length; i++) {
                         const property_details = fm_properties[i].split(':', 2);
                         if (property_details.length > 1) {
-                            response.fm_properties.push({key: property_details[0].trim(), value: property_details[1].trim()});
+                            response.fm_properties[property_details[0].trim().toLowerCase()] = property_details[1].trim();
                         }
                     }
 

package/hdoc.js

@@ -1,4 +1,4 @@
-#!/usr/bin/env node
+#!/usr/bin/env node
 
 (function () {
    'use strict';
@@ -77,9 +77,6 @@
    } else if (command == 'help') {
       const help = require(path.join(__dirname, 'hdoc-help.js'));
       help.run();
-   } else if (command == 'buildindex') {
-      const build_db = require(path.join(__dirname, 'hdoc-db.js'));
-      build_db.run(source_path);
    } else {
       console.log('Unknown command:', command, '\r\n');
       console.log('Run hdoc help for information regarding this tool.\r\n');

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "hdoc-tools",
-  "version": "0.7.27",
+  "version": "0.8.0",
   "description": "Hornbill HDocBook Development Support Tool",
   "main": "hdoc.js",
   "bin": {
@@ -28,6 +28,7 @@
   "author": "Hornbill Technologies Ltd",
   "license": "ISC",
   "dependencies": {
+    "better-sqlite3": "^8.0.1",
     "body-parser": "^1.20.1",
     "cheerio": "^1.0.0-rc.12",
     "cookie-parser": "^1.4.6",
@@ -42,7 +43,6 @@
     "markdown-it-front-matter": "^0.2.3",
     "multer": "^1.4.5-lts.1",
     "prompt": "^1.3.0",
-    "sqlite3": "^5.1.4",
     "stream": "0.0.2",
     "sync-request": "^6.1.0",
     "words-count": "^2.0.2",