hdoc-tools 0.20.0 → 0.22.0

package/hdoc-build-db.js

@@ -1,260 +1,262 @@
-(() => {
-	const path = require("node:path");
-	const hdoc_index = require(path.join(__dirname, "hdoc-db.js"));
-	const database = require("better-sqlite3");
-
-	const db_schema = {
-		hdoc_index: [
-			"resource_url UNINDEXED",
-			"book_id",
-			"book_audience UNINDEXED",
-			"book_tags",
-			"doc_title",
-			"doc_content",
-			"doc_preview UNINDEXED",
-			"doc_family_id",
-			"doc_md5_hash UNINDEXED",
-			"doc_lastmod",
-		],
-		hdoc_meta: [
-			"resource_url",
-			"book_id",
-			"contributor_count INTEGER UNINDEXED",
-			"edit_url UNINDEXED",
-			"last_commit UNINDEXED",
-			"pdf_size INTEGER UNINDEXED",
-		],
-		hdoc_contributors: [
-			"resource_url",
-			"book_id",
-			"login",
-			"name",
-			"avatar UNINDEXED",
-			"url UNINDEXED",
-		],
-		hdoc_redirects: [
-			"resource_url",
-			"location_url",
-			"http_code INTEGER UNINDEXED",
-		],
-	};
-
-	exports.create_table = (db, table, cols, virtual, fts5) => {
-		const table_created = hdoc_index.create_table(
-			db,
-			table,
-			cols,
-			virtual,
-			fts5,
-		);
-		if (table_created !== null) {
-			return `\nError creating table: ${table_created}`;
-		}
-		console.log(`\nTable created: ${table}`);
-		return null;
-	};
-
-	exports.create_db = function (db_path, doc_id) {
-		const response = {
-			error: null,
-			db: null,
-		};
-		console.log("\nPerforming SQlite index creation...");
-		const db_name = path.join(db_path, doc_id, "index.db");
-		response.db = new database(db_name);
-		response.db.pragma('encoding="UTF-8"');
-		console.log(`\nDatabase created: ${db_name}`);
-
-		// Create tables
-		for (const key in db_schema) {
-			if (Object.hasOwn(db_schema, key)) {
-				const virtual = key === "hdoc_index";
-				const fts5 = key === "hdoc_index";
-				const table_created_error = this.create_table(
-					response.db,
-					key,
-					db_schema[key],
-					virtual,
-					fts5,
-				);
-				if (table_created_error !== null) {
-					console.error(table_created_error);
-				}
-			}
-		}
-		return response;
-	};
-
-	exports.populate_redirects = (db, redirect_records, verbose = false) => {
-		const response = {
-			success: true,
-			errors: [],
-			index_success_count: 0,
-		};
-
-		for (let i = 0; i < redirect_records.length; i++) {
-			const index_vals = [
-				redirect_records[i].url,
-				redirect_records[i].location ? redirect_records[i].location : "",
-				redirect_records[i].code,
-			];
-			const index_response = hdoc_index.insert_record(
-				db,
-				"hdoc_redirects",
-				db_schema.hdoc_redirects,
-				index_vals,
-			);
-			if (!index_response.success) {
-				response.success = false;
-				response.errors.push(
-					`Redirect record creation failed - ${redirect_records[i].url}: ${index_response.error}`,
-				);
-			} else {
-				response.index_success_count++;
-			}
-		}
-		console.log(
-			`\nRedirect Index Build Complete: ${response.index_success_count} document records created.`,
-		);
-		return response;
-	};
-
-	exports.populate_index = async (
-		db,
-		doc_id,
-		book_config,
-		index_records,
-		verbose = false,
-	) => {
-		const response = {
-			success: false,
-			index_success_count: 0,
-			error: "",
-		};
-
-		if (!book_config.tags) book_config.tags = [];
-
-		const indexPromises = [];
-		for (let i = 0; i < index_records.length; i++) {
-			indexPromises.push(index_records[i]);
-		}
-		let curr_file = "";
-		await Promise.all(
-			indexPromises.map(async (file) => {
-				let index_path_name = file.relative_path.replaceAll("\\", "/");
-				if (
-					index_path_name.endsWith("/index.md") ||
-					index_path_name.endsWith("/index.html") ||
-					index_path_name.endsWith("/index.htm")
-				) {
-					index_path_name = index_path_name.substring(
-						0,
-						index_path_name.lastIndexOf("/"),
-					);
-				}
-				index_path_name = `/${index_path_name.replace(path.extname(file.relative_path), "")}`;
-
-				let index_response = {
-					success: true,
-					row_id: 0,
-				};
-				let index_content_path = index_path_name;
-				if (file.index_html.id !== null)
-					index_content_path += `#${file.index_html.id}`;
-				if (!file.inline) {
-					const index_vals = [
-						index_content_path,
-						doc_id,
-						book_config.audience.join(","),
-						book_config.tags.join(","),
-						file.index_html.fm_props.title,
-						file.index_html.text,
-						file.index_html.preview,
-						book_config.productFamily,
-						file.md5,
-						file.lastmod,
-					];
-					index_response = hdoc_index.insert_record(
-						db,
-						"hdoc_index",
-						db_schema.hdoc_index,
-						index_vals,
-					);
-				}
-				if (!index_response.success) {
-					console.error(
-						`Index record creation failed - ${doc_id}/${file.index_html.fm_props.title}: ${index_response.error}`,
-					);
-				} else {
-					if (curr_file === index_path_name) return;
-					curr_file = index_path_name;
-					// Now add metadata
-					const meta_vals = [
-						index_path_name,
-						doc_id,
-						file.metadata.contributor_count,
-						file.metadata.edit_url,
-						file.metadata.last_commit,
-						file.pdf_size,
-					];
-					const meta_response = await hdoc_index.insert_record(
-						db,
-						"hdoc_meta",
-						db_schema.hdoc_meta,
-						meta_vals,
-					);
-					if (!meta_response.success) {
-						console.error(
-							`Index metadata record creation failed - ${doc_id}/${index_response.row_id}/${file.index_html.fm_props.title}: ${meta_response.error}`,
-						);
-					} else {
-						if (verbose) {
-							console.log(
-								`Inserted index record ${index_response.row_id}: ${doc_id} - ${file.index_html.fm_props.title}`,
-							);
-							console.log(
-								`Inserted index metadata record for index ID: ${meta_response.row_id}`,
-							);
-						}
-
-						// Now add contributor records
-						for (let j = 0; j < file.contributors.length; j++) {
-							const contrib_vals = [
-								index_path_name,
-								doc_id,
-								file.contributors[j].login,
-								file.contributors[j].name,
-								file.contributors[j].avatar_url,
-								file.contributors[j].html_url,
-							];
-							const cont_response = await hdoc_index.insert_record(
-								db,
-								"hdoc_contributors",
-								db_schema.hdoc_contributors,
-								contrib_vals,
-							);
-							if (!cont_response.success) {
-								console.error(
-									`Index document contributor record creation failed - ${doc_id}/${index_response.row_id}/${file.index_html.fm_props.title}: ${cont_response.error}`,
-								);
-								continue;
-							}
-							if (verbose) {
-								console.log(
-									`Inserted document contributor record ${cont_response.row_id}`,
-								);
-							}
-						}
-						response.index_success_count++;
-					}
-				}
-			}),
-		);
-
-		response.success = true;
-		console.log(
-			`\nIndex Build Complete: ${response.index_success_count} document index records created.`,
-		);
-		return response;
-	};
-})();
+(() => {
+	const path = require("node:path");
+	const hdoc_index = require(path.join(__dirname, "hdoc-db.js"));
+	const database = require("better-sqlite3");
+
+	const db_schema = {
+		hdoc_index: [
+			"resource_url UNINDEXED",
+			"book_id",
+			"book_audience UNINDEXED",
+			"book_tags",
+			"doc_title",
+			"doc_content",
+			"doc_preview UNINDEXED",
+			"doc_family_id",
+			"doc_md5_hash UNINDEXED",
+			"doc_lastmod",
+			"doc_status",
+		],
+		hdoc_meta: [
+			"resource_url",
+			"book_id",
+			"contributor_count INTEGER UNINDEXED",
+			"edit_url UNINDEXED",
+			"last_commit UNINDEXED",
+			"pdf_size INTEGER UNINDEXED",
+		],
+		hdoc_contributors: [
+			"resource_url",
+			"book_id",
+			"login",
+			"name",
+			"avatar UNINDEXED",
+			"url UNINDEXED",
+		],
+		hdoc_redirects: [
+			"resource_url",
+			"location_url",
+			"http_code INTEGER UNINDEXED",
+		],
+	};
+
+	exports.create_table = (db, table, cols, virtual, fts5) => {
+		const table_created = hdoc_index.create_table(
+			db,
+			table,
+			cols,
+			virtual,
+			fts5,
+		);
+		if (table_created !== null) {
+			return `\nError creating table: ${table_created}`;
+		}
+		console.log(`\nTable created: ${table}`);
+		return null;
+	};
+
+	exports.create_db = function (db_path, doc_id) {
+		const response = {
+			error: null,
+			db: null,
+		};
+		console.log("\nPerforming SQlite index creation...");
+		const db_name = path.join(db_path, doc_id, "index.db");
+		response.db = new database(db_name);
+		response.db.pragma('encoding="UTF-8"');
+		console.log(`\nDatabase created: ${db_name}`);
+
+		// Create tables
+		for (const key in db_schema) {
+			if (Object.hasOwn(db_schema, key)) {
+				const virtual = key === "hdoc_index";
+				const fts5 = key === "hdoc_index";
+				const table_created_error = this.create_table(
+					response.db,
+					key,
+					db_schema[key],
+					virtual,
+					fts5,
+				);
+				if (table_created_error !== null) {
+					console.error(table_created_error);
+				}
+			}
+		}
+		return response;
+	};
+
+	exports.populate_redirects = (db, redirect_records, verbose = false) => {
+		const response = {
+			success: true,
+			errors: [],
+			index_success_count: 0,
+		};
+
+		for (let i = 0; i < redirect_records.length; i++) {
+			const index_vals = [
+				redirect_records[i].url,
+				redirect_records[i].location ? redirect_records[i].location : "",
+				redirect_records[i].code,
+			];
+			const index_response = hdoc_index.insert_record(
+				db,
+				"hdoc_redirects",
+				db_schema.hdoc_redirects,
+				index_vals,
+			);
+			if (!index_response.success) {
+				response.success = false;
+				response.errors.push(
+					`Redirect record creation failed - ${redirect_records[i].url}: ${index_response.error}`,
+				);
+			} else {
+				response.index_success_count++;
+			}
+		}
+		console.log(
+			`\nRedirect Index Build Complete: ${response.index_success_count} document records created.`,
+		);
+		return response;
+	};
+
+	exports.populate_index = async (
+		db,
+		doc_id,
+		book_config,
+		index_records,
+		verbose = false,
+	) => {
+		const response = {
+			success: false,
+			index_success_count: 0,
+			error: "",
+		};
+
+		if (!book_config.tags) book_config.tags = [];
+
+		const indexPromises = [];
+		for (let i = 0; i < index_records.length; i++) {
+			indexPromises.push(index_records[i]);
+		}
+		let curr_file = "";
+		await Promise.all(
+			indexPromises.map(async (file) => {
+				let index_path_name = file.relative_path.replaceAll("\\", "/");
+				if (
+					index_path_name.endsWith("/index.md") ||
+					index_path_name.endsWith("/index.html") ||
+					index_path_name.endsWith("/index.htm")
+				) {
+					index_path_name = index_path_name.substring(
+						0,
+						index_path_name.lastIndexOf("/"),
+					);
+				}
+				index_path_name = `/${index_path_name.replace(path.extname(file.relative_path), "")}`;
+
+				let index_response = {
+					success: true,
+					row_id: 0,
+				};
+				let index_content_path = index_path_name;
+				if (file.index_html.id !== null)
+					index_content_path += `#${file.index_html.id}`;
+				if (!file.inline) {
+					const index_vals = [
+						index_content_path,
+						doc_id,
+						book_config.audience.join(","),
+						book_config.tags.join(","),
+						file.index_html.fm_props.title,
+						file.index_html.text,
+						file.index_html.preview,
+						book_config.productFamily,
+						file.md5,
+						file.lastmod,
+						file.status,
+					];
+					index_response = hdoc_index.insert_record(
+						db,
+						"hdoc_index",
+						db_schema.hdoc_index,
+						index_vals,
+					);
+				}
+				if (!index_response.success) {
+					console.error(
+						`Index record creation failed - ${doc_id}/${file.index_html.fm_props.title}: ${index_response.error}`,
+					);
+				} else {
+					if (curr_file === index_path_name) return;
+					curr_file = index_path_name;
+					// Now add metadata
+					const meta_vals = [
+						index_path_name,
+						doc_id,
+						file.metadata.contributor_count,
+						file.metadata.edit_url,
+						file.metadata.last_commit,
+						file.pdf_size,
+					];
+					const meta_response = await hdoc_index.insert_record(
+						db,
+						"hdoc_meta",
+						db_schema.hdoc_meta,
+						meta_vals,
+					);
+					if (!meta_response.success) {
+						console.error(
+							`Index metadata record creation failed - ${doc_id}/${index_response.row_id}/${file.index_html.fm_props.title}: ${meta_response.error}`,
+						);
+					} else {
+						if (verbose) {
+							console.log(
+								`Inserted index record ${index_response.row_id}: ${doc_id} - ${file.index_html.fm_props.title}`,
+							);
+							console.log(
+								`Inserted index metadata record for index ID: ${meta_response.row_id}`,
+							);
+						}
+
+						// Now add contributor records
+						for (let j = 0; j < file.contributors.length; j++) {
+							const contrib_vals = [
+								index_path_name,
+								doc_id,
+								file.contributors[j].login,
+								file.contributors[j].name,
+								file.contributors[j].avatar_url,
+								file.contributors[j].html_url,
+							];
+							const cont_response = await hdoc_index.insert_record(
+								db,
+								"hdoc_contributors",
+								db_schema.hdoc_contributors,
+								contrib_vals,
+							);
+							if (!cont_response.success) {
+								console.error(
+									`Index document contributor record creation failed - ${doc_id}/${index_response.row_id}/${file.index_html.fm_props.title}: ${cont_response.error}`,
+								);
+								continue;
+							}
+							if (verbose) {
+								console.log(
+									`Inserted document contributor record ${cont_response.row_id}`,
+								);
+							}
+						}
+						response.index_success_count++;
+					}
+				}
+			}),
+		);
+
+		response.success = true;
+		console.log(
+			`\nIndex Build Complete: ${response.index_success_count} document index records created.`,
+		);
+		return response;
+	};
+})();

package/hdoc-build.js

@@ -5,7 +5,6 @@
 	const mdfm = require("markdown-it-front-matter");
 	const path = require("node:path");
 	const puppeteer = require("puppeteer");
-	const URL = require("node:url").URL;
 	const hdoc_validate = require(path.join(__dirname, "hdoc-validate.js"));
 	const hdoc = require(path.join(__dirname, "hdoc-module.js"));
 	const hdoc_build_db = require(path.join(__dirname, "hdoc-build-db.js"));
@@ -45,6 +44,7 @@
 
 	const built_file_hashes = [];
 	const css_templates = [];
+	const draft_links = [];
 	const errors_filename = [];
 	const index_records = [];
 	const md_files = [];
@@ -119,9 +119,18 @@
 
 			// Check if we have a frontmatter comment
 			const fm_header = hdoc.getHTMLFrontmatterHeader(html_txt);
+
+			let rel_path = file_path.relativePath.replace(path.extname(file_path.relativePath), '');
+			if (rel_path.endsWith('/index')) rel_path = rel_path.substring(0, rel_path.length - 6);
+			const is_draft = draft_links.indexOf(rel_path) !== -1;
+
+			let fm_status = false;
 			if (Object.keys(fm_header.fm_properties).length > 0) {
 				existing_fm_headers = true;
 
+				if (fm_header.fm_properties?.status) {
+					fm_status = true;
+				}
 				// We have some frontmatter headers, check if title is one of them
 				let fm_title_found = false;
 				if (
@@ -268,6 +277,14 @@
 				value: doc_type,
 			});
 
+			// Add status if we don't have one defined and the nav tree defines the document as draft
+			if (!fm_status && is_draft) {
+				fm_headers.push({
+					id: "status",
+					value: "draft"
+				});
+			}
+
 			const metadata = {};
 
 			// Remove the first <h1>title</h1> from the HTML as we'll add that in the document header
@@ -443,6 +460,7 @@
 					md5: file_path.hash,
 					lastmod: last_commit !== null ? last_commit : file_path.hb_lastmod,
 					inline: inline_content,
+					status: index_data.fm_props.status ? index_data.fm_props.status : 'release',
 				});
 			}
 
@@ -506,7 +524,6 @@
 			});
 
 			// Tidy up ```json and ```xml code tags
-
 			if (md_txt.includes("```json") || md_txt.includes("```xml"))
 				md_txt = tidy_code_tags(md_txt, file_path.relativePath);
 
@@ -523,6 +540,11 @@
 			let doc_title = "";
 			let doc_type = "Article";
 
+			let rel_path = file_path.relativePath.replace(path.extname(file_path.relativePath), '');
+			if (rel_path.endsWith('/index')) rel_path = rel_path.substring(0, rel_path.length - 6);
+			const is_draft = draft_links.indexOf(rel_path) !== -1;
+			let fm_status = false;
+
 			if (fm_content.length >= 0) {
 				for (fm_prop of fm_content) {
 					const fm_id = fm_prop.slice(0, fm_prop.indexOf(":"));
@@ -543,6 +565,9 @@
 							fm_contains_title = true;
 							doc_title = fm_val.trim();
 						}
+						if (fm_id.trim() === "status") {
+							fm_status = true;
+						}
 						if (fm_id.trim() === "type") {
 							doc_type = fm_val.trim();
 						}
@@ -563,6 +588,14 @@
 				value: doc_type,
 			});
 
+			// Add status if we don't have one defined and the nav tree defines the document as draft
+			if (!fm_status && is_draft) {
+				fm_headers.push({
+					id: "status",
+					value: "draft"
+				});
+			}
+
 			// Does frontmatter tag contain a title property
 			if (!fm_contains_title) {
 				// Frontmatter tags don't contain a title property - go pull the first one from the html heading tags
@@ -791,7 +824,6 @@
 			}
 
 			const index_data = hdoc_index.transform_html_for_index(html_txt);
-
 			for (section of index_data.sections) {
 				index_records.push({
 					relative_path: relative_path,
@@ -807,6 +839,7 @@
 					md5: file_path.hash,
 					lastmod: last_commit !== null ? last_commit : file_path.hb_lastmod,
 					inline: inline_content,
+					status: index_data.fm_props.status ? index_data.fm_props.status : 'release',
 				});
 			}
 
@@ -1269,6 +1302,10 @@
 		}
 		console.log("Processing navigation breadcrumbs...");
 		const bc_build = hdoc.build_breadcrumbs(hdocbook_config.navigation.items);
+
+		console.log("Getting list of draft links to exclude from the build...");
+		draft_links.push(...hdoc.get_draft_links(hdocbook_config.navigation.items));
+
 		if (bc_build.errors.length > 0) {
 			console.log("\r\n-----------------------");
 			console.log("   Validation Output   ");
@@ -1282,6 +1319,7 @@
 		}
 		bc = bc_build.bc;
 		console.log("Processing content...");
+
 		// Get a list of MD files in work_path
 		dree.scan(work_path, dreeOptions, build_file_callback);
 
@@ -1355,6 +1393,7 @@
 			prods_supported,
 			gen_exclude,
 			redirects,
+			draft_links,
 		);
 		if (!validation_success) {
 			const end_time = Date.now();