hdoc-tools 0.45.1 → 0.47.0

package/hdoc-build.js

@@ -1,7 +1,6 @@
 (() => {
 	const crypto = require("node:crypto");
-	const dree = require("dree");
-	const fs = require("fs-extra");
+	const fs = require("node:fs");
 	const mdfm = require("markdown-it-front-matter");
 	const path = require("node:path");
 	const puppeteer = require("puppeteer");
@@ -11,10 +10,8 @@
 	const hdoc_build_pdf = require(path.join(__dirname, "hdoc-build-pdf.js"));
 	const hdoc_index = require(path.join(__dirname, "hdoc-db.js"));
 	const archiver = require("archiver");
-	const xmlFormat = require("xml-formatter");
 
 	const { execSync } = require("child_process");
-	const tmp = require("tmp");
 
 	const h_tags_to_search = ["h1", "h2", "h3"];
 	const image_extensions = ["png", "svg", "jpg"];
@@ -101,6 +98,71 @@
 	let mermaid_images_path = "";
 	let verbose = false;
 
+	// Mutable references updated immediately before each md.render() call so
+	// the highlight and frontmatter plugin callbacks can identify the current
+	// file. Safe because md.render() is synchronous — nothing can interleave
+	// between setting these and reading them back within a single render.
+	let currentMdFilePath = "";
+	let currentFrontmatter = "";
+
+	// Shared markdown-it instance — created once, reused for every file.
+	// Previously recreated per file because the highlight callback closed over
+	// the file_path parameter; it now reads currentMdFilePath instead.
+	// mdfm and tips are registered once here rather than once per file.
+	const md = require("markdown-it")({
+		html: true,
+		linkify: true,
+		typographer: true,
+		highlight: function (str, lang) {
+			if (lang === "mermaid" && process.env.GITHUB_ACTIONS !== 'true') {
+				try {
+					const tmpInput = hdoc.tmp_file_sync({ postfix: ".mmd" });
+					const outputFileName = `mermaid-${crypto.createHash("sha256").update(str).digest("hex").slice(0, 16)}.svg`;
+					const outputPath = path.join(mermaid_images_path, outputFileName);
+					const outputLink = `/_books/${doc_id}/mermaid-images/${outputFileName}`;
+
+					if (!str.startsWith('---')) {
+						str = '---\n' + fs.readFileSync(mermaid_theme_path, {encoding: 'utf-8'}) + `\n---\n${str}`;
+					}
+
+					fs.writeFileSync(tmpInput.name, str);
+					let cmd = `${__dirname}/node_modules/.bin/mmdc`;
+
+					if (process.platform === "win32") {
+						cmd = `"${cmd}.cmd"`;
+					}
+
+					cmd = `${cmd} -i "${tmpInput.name}" -o "${outputPath}" --backgroundColor transparent --puppeteerConfigFile ${mermaid_puppeteer_config_path}`;
+					console.log(`Generating Mermaid SVG found in ${currentMdFilePath.relativePath} - ${outputPath}`);
+					execSync(cmd);
+
+					if (!fs.existsSync(outputPath)) {
+						throw new Error("mmdc did not generate output");
+					}
+
+					tmpInput.removeCallback();
+
+					return `<img class="mermaid-diagram" src="${outputLink}" alt="Mermaid Diagram">`;
+				} catch (err) {
+					mermaid_failures.push({path: currentMdFilePath.relativePath, error: err.message});
+					return ``;
+				}
+			}
+		}
+	});
+	md.linkify.set({
+		fuzzyEmail: false,
+		fuzzyLink: false,
+		fuzzyIP: false,
+	});
+	// mdfm callback writes to currentFrontmatter, which is reset to "" before
+	// each render so every file gets a clean slate.
+	md.use(mdfm, (fm) => {
+		currentFrontmatter = fm;
+	});
+	const tips = require(`${__dirname}/custom_modules/tips.js`);
+	md.use(tips, { links: true });
+
 	const pdf_path_excluded = (relative_path) => {
 		if (
 			!hdocbook_project.pdfGeneration ||
@@ -128,62 +190,202 @@
 		return false;
 	};
 
-	const transform_static_html = async (file_path) => {
-		if (fs.existsSync(file_path.path)) {
+	// Processes a single documentation file (markdown or static HTML), generates
+	// frontmatter headers, retrieves contributor data from GitHub, optionally
+	// generates a PDF, and writes the final HTML to disk ready for indexing.
+	// Markdown files are rendered to HTML first; static HTML files are processed
+	// in place. Both paths share the same post-processing pipeline.
+	const transform_file = async (file_path) => {
+		const is_markdown = path.extname(file_path.path) === '.md';
+		if (is_markdown) conversion_attempted++;
+
+		if (!fs.existsSync(file_path.path)) {
+			if (is_markdown) {
+				conversion_failed++;
+				console.error("MD file does not exist:", file_path.path);
+				return false;
+			}
+			return;
+		}
+
+		let html_txt;
+		const fm_headers = [];
+		let doc_title = "";
+		let doc_type = "Article";
+		let fm_status = false;
+		// Used only for static HTML: tracks whether the source file already had a
+		// frontmatter comment block that needs replacing in the output.
+		let existing_fm_headers = false;
+		let html_fm = null;
+
+		if (is_markdown) {
+			// Load markdown file
+			let md_txt = hdoc.expand_variables(
+				fs.readFileSync(file_path.path, "utf8"),
+			);
+
+			// Pull in external includes
+			const includes_processed = await hdoc.process_includes(
+				file_path.path,
+				md_txt,
+				global_source_path,
+			);
+			md_txt = includes_processed.body.toString();
+			includes_found += includes_processed.found;
+			includes_success += includes_processed.success;
+			includes_failed += includes_processed.failed;
+			if (includes_processed.errors.length > 0) {
+				for (let i = 0; i < includes_processed.errors.length; i++) {
+					console.error(includes_processed.errors[i]);
+				}
+			}
+
+			// Point the shared md instance at the current file before rendering
+			// so the highlight callback logs and reports against the right path.
+			currentMdFilePath = file_path;
+			currentFrontmatter = "";
+
+			// Tidy up ```json and ```xml code tags
+			if (md_txt.includes("```json") || md_txt.includes("```xml"))
+				md_txt = tidy_code_tags(md_txt, file_path.relativePath);
+
+			// Render markdown into HTML
+			html_txt = md.render(md_txt);
+
+			// Parse frontmatter properties from the YAML block
+			let fm_contains_title = false;
+			let fm_contains_reading_time = false;
+			let fm_contains_description = false;
+
+			const fm_content = currentFrontmatter.split(/\r?\n/);
+			if (fm_content.length >= 0) {
+				for (fm_prop of fm_content) {
+					const fm_id = fm_prop.slice(0, fm_prop.indexOf(":"));
+					const fm_val = fm_prop.slice(fm_prop.indexOf(":") + 1);
+
+					if (
+						fm_id &&
+						fm_id.trim().length > 0 &&
+						fm_val &&
+						fm_val.trim().length > 0
+					) {
+						fm_headers.push({
+							id: fm_id.trim(),
+							value: fm_val.trim(),
+						});
+
+						if (fm_id.trim() === "title") {
+							fm_contains_title = true;
+							doc_title = fm_val.trim();
+						}
+						if (fm_id.trim() === "status") {
+							fm_status = true;
+						}
+						if (fm_id.trim() === "type") {
+							doc_type = fm_val.trim();
+						}
+						if (fm_id.trim() === "reading-time") {
+							book_read_time += Number.parseInt(fm_val.trim(), 10);
+							fm_contains_reading_time = true;
+						}
+						if (fm_id.trim() === "description") {
+							fm_contains_description = true;
+						}
+					}
+				}
+			}
+
+			// Title from heading if not in frontmatter
+			if (!fm_contains_title) {
+				const html_heading = hdoc.getFirstHTMLHeading(
+					html_txt,
+					h_tags_to_search,
+				);
+
+				if (html_heading?.[0]?.children?.[0]?.data) {
+					fm_headers.push({
+						id: "title",
+						value: html_heading[0].children[0].data.trim(),
+					});
+					doc_title = html_heading[0].children[0].data.trim();
+				} else if (
+					file_path.name !== "description_ext.md" &&
+					file_path.name !== "article_ext.md" &&
+					file_path.name !== "internal_ext.md"
+				) {
+					console.info(
+						`[WARNING] No frontmatter title property, or ${h_tags_to_search.join(", ")} tags detected in ${file_path.path}`,
+					);
+				}
+			}
+
+			// Description from first paragraph if not in frontmatter
+			if (!fm_contains_description) {
+				const html_p_tag = hdoc.getFirstHTMLHeading(html_txt, ["p"]);
+				if (html_p_tag?.[0]?.children?.[0]?.data) {
+					fm_headers.push({
+						id: "description",
+						value:
+							`${doc_title}: ${html_p_tag[0].children[0].data.split(".")[0]}.`.trim(),
+					});
+				}
+			}
+
+			// Reading time from content if not in frontmatter
+			if (!fm_contains_reading_time) {
+				const read_time_mins = hdoc.get_html_read_time(html_txt);
+				book_read_time += read_time_mins;
+				fm_headers.push({
+					id: "reading-time",
+					value: read_time_mins,
+				});
+			}
+		} else {
 			// Load HTML file
-			let html_txt = fs.readFileSync(file_path.path, "utf8");
+			html_txt = fs.readFileSync(file_path.path, "utf8");
 			html_txt = html_txt.replace(/\r/gm, ""); // Remove CR's so we're just dealing with newlines
 
-			const fm_headers = [];
-			let existing_fm_headers = false;
-			let doc_type = "Article";
-			let doc_title = "";
-
 			// Check if we have a frontmatter comment
-			const fm_header = hdoc.getHTMLFrontmatterHeader(html_txt);
+			html_fm = hdoc.getHTMLFrontmatterHeader(html_txt);
 
-			let rel_path = file_path.relativePath.replace(path.extname(file_path.relativePath), '');
-			if (rel_path.endsWith('/index')) rel_path = rel_path.substring(0, rel_path.length - 6);
-			const is_draft = draft_links.indexOf(rel_path) !== -1;
-
-			let fm_status = false;
-			if (Object.keys(fm_header.fm_properties).length > 0) {
+			if (Object.keys(html_fm.fm_properties).length > 0) {
 				existing_fm_headers = true;
 
-				if (fm_header.fm_properties?.status) {
+				if (html_fm.fm_properties?.status) {
 					fm_status = true;
 				}
+
 				// We have some frontmatter headers, check if title is one of them
 				let fm_title_found = false;
 				if (
-					fm_header.fm_properties &&
-					fm_header.fm_properties.title !== undefined
+					html_fm.fm_properties &&
+					html_fm.fm_properties.title !== undefined
 				) {
 					// We have a title - but does the title have a value
-					if (fm_header.fm_properties.title === "") {
-						// No value - remove title from the properties map  so we don't end up with 2 title properties, one empty and one with a value
-						fm_header.fm_properties.title = undefined;
+					if (html_fm.fm_properties.title === "") {
+						// No value - remove title from the properties map so we don't end up with 2 title properties, one empty and one with a value
+						html_fm.fm_properties.title = undefined;
 					} else {
 						// We have a value for the title property
 						fm_title_found = true;
-						doc_title = fm_header.fm_properties.title.trim();
+						doc_title = html_fm.fm_properties.title.trim();
 					}
 				}
 
 				// Is reading-time in the fm headers?
-				if (fm_header.fm_properties["reading-time"] === undefined) {
+				if (html_fm.fm_properties["reading-time"] === undefined) {
 					const read_time_mins = hdoc.get_html_read_time(html_txt);
 					book_read_time += read_time_mins;
-					fm_header.fm_properties["reading-time"] = read_time_mins;
+					html_fm.fm_properties["reading-time"] = read_time_mins;
 				}
 
-				for (const key in fm_header.fm_properties) {
-					if (Object.hasOwn(fm_header.fm_properties, key)) {
-						if (key === "type") doc_type = fm_header.fm_properties[key];
+				for (const key in html_fm.fm_properties) {
+					if (Object.hasOwn(html_fm.fm_properties, key)) {
+						if (key === "type") doc_type = html_fm.fm_properties[key];
 						else {
 							fm_headers.push({
 								id: key,
-								value: fm_header.fm_properties[key],
+								value: html_fm.fm_properties[key],
 							});
 						}
 					}
@@ -192,10 +394,10 @@
 				if (
 					!fm_title_found &&
 					file_path.name !== "description_ext.md" &&
-					file_path.name !== "article_ext.md"
+					file_path.name !== "article_ext.md" &&
+					file_path.name !== "internal_ext.md"
 				) {
-					// No frontmatter title found in properties
-					// Go get title from h tags in html
+					// No frontmatter title found in properties - go get title from h tags in html
 					const html_heading = hdoc.getFirstHTMLHeading(
 						html_txt,
 						h_tags_to_search,
@@ -220,10 +422,10 @@
 
 				// Do we have a description header?
 				if (
-					fm_header.fm_properties &&
-					fm_header.fm_properties.description !== undefined
+					html_fm.fm_properties &&
+					html_fm.fm_properties.description !== undefined
 				) {
-					if (fm_header.fm_properties.description === "") {
+					if (html_fm.fm_properties.description === "") {
 						const html_p_tag = hdoc.getFirstHTMLHeading(html_txt, ["p"]);
 						if (html_p_tag?.[0]?.children?.[0]?.data) {
 							fm_headers.push({
@@ -235,7 +437,7 @@
 					} else {
 						fm_headers.push({
 							id: "description",
-							value: fm_header.fm_properties.description.trim(),
+							value: html_fm.fm_properties.description.trim(),
 						});
 					}
 				} else {
@@ -254,20 +456,20 @@
 					html_txt,
 					h_tags_to_search,
 				);
-				let doc_title = "";
+				let doc_title_local = "";
 				// Add the title
 				if (html_heading?.[0]?.children?.[0]?.data) {
-					// We've found a heading tag, add that as a title to the frontmatter content
 					fm_headers.push({
 						id: "title",
 						value: html_heading[0].children[0].data,
 					});
-					doc_title = html_heading[0].children[0].data;
+					doc_title_local = html_heading[0].children[0].data;
+					doc_title = doc_title_local;
 				} else if (
 					file_path.name !== "description_ext.md" &&
-					file_path.name !== "article_ext.md"
+					file_path.name !== "article_ext.md" &&
+					file_path.name !== "internal_ext.md"
 				) {
-					// No header tag, no frontmatter title, output a warning
 					console.info(
 						`[WARNING] No frontmatter title property, or ${h_tags_to_search.join(
 							", ",
@@ -288,633 +490,244 @@
 					fm_headers.push({
 						id: "description",
 						value:
-							`${doc_title}: ${html_p_tag[0].children[0].data.split(".")[0]}.`.trim(),
-					});
-				}
-			}
-
-			// Add doc type
-			fm_headers.push({
-				id: "type",
-				value: doc_type,
-			});
-
-			// Add status if we don't have one defined and the nav tree defines the document as draft
-			if (!fm_status && is_draft) {
-				fm_headers.push({
-					id: "status",
-					value: "draft"
-				});
-			}
-
-			const metadata = {};
-
-			// Remove the first <h1>title</h1> from the HTML as we'll add that in the document header
-			let html_h1 = h1_pattern.exec(html_txt);
-			if (html_h1?.[0])
-				html_h1 = html_h1[0].replace(/(<h1.*?>)/, "").replace(/(<\/h1>)/, "");
-
-			html_txt = html_txt.replace(h1_pattern, "");
-
-			// Get contributor data from Github, if exists
-			let contribs = [];
-			let last_commit = null;
-			if (
-				hdocbook_config.publicSource &&
-				hdocbook_config.publicSource !== "" &&
-				hdocbook_config.publicSource.includes("github.com/Hornbill-Docs")
-			) {
-				const github_paths = hdoc.get_github_api_path(
-					hdocbook_config.publicSource,
-					file_path.relativePath,
-				);
-				
-				const contributors = await hdoc.get_github_contributors(
-					github_paths.api_path,
-					git_token,
-				);
-
-				if (!contributors.success) {
-					console.error(
-						`Error retrieving contributors from Github: ${contributors.error}`,
-					);
-				} else {
-					last_commit = contributors.last_commit_date;
-					metadata.last_commit = contributors.last_commit_date;
-					metadata.contributor_count = contributors.contributor_count;
-					metadata.edit_url = github_paths.edit_path;
-					contribs = contributors.contributors;
-					contributors.editPath = github_paths.edit_path;
-					fm_headers.push({
-						id: "contributor-count",
-						value: contributors.contributor_count,
+							`${doc_title_local}: ${html_p_tag[0].children[0].data.split(".")[0]}.`.trim(),
 					});
-					fm_headers.push({
-						id: "last-commit",
-						value: contributors.last_commit_date,
-					});
-					const target_file = file_path.path.replace(
-						path.extname(file_path.path),
-						"._info.json",
-					);
-					contributors.success = undefined;
-					contributors.error = undefined;
-					contributors.editPath = github_paths.edit_path;
-				}
-				fm_headers.push({
-					id: "edit-path",
-					value: github_paths.edit_path,
-				});
-			}
-
-			if (pdf_enable && !pdf_path_excluded(file_path.relativePath)) {
-				fm_headers.push({
-					id: "pdf-path",
-					value: file_path.relativePath.replace(
-						path.extname(file_path.relativePath),
-						".pdf",
-					),
-				});
-			}
-
-			let fm_header_content = "<!--[[FRONTMATTER\n";
-			if (fm_headers.length > 0) {
-				for (let i = 0; i < fm_headers.length; i++) {
-					fm_header_content += `${fm_headers[i].id}: ${fm_headers[i].value}\n`;
-				}
-				fm_header_content += "]]-->";
-
-				if (existing_fm_headers) {
-					html_txt = html_txt.replace(`<!--${fm_header.fm_header}-->`, "");
 				}
 			}
-
-			let doc_header = "";
-			let pdf_header = "";
-			const inline_content = file_path.relativePath.startsWith(
-				`${hdocbook_config.docId}/_inline/`,
-			);
-			if (hdocbook_config.publicSource?.includes("github.com/Hornbill-Docs")) {
-				// Build doc header from template and frontmatter tags
-				if (!inline_content)
-					doc_header = process_doc_header(
-						fm_headers,
-						file_path.relativePath,
-						doc_header_template,
-						html_h1,
-					);
-
-				if (pdf_enable && !pdf_path_excluded(file_path.relativePath))
-					pdf_header = process_doc_header(
-						fm_headers,
-						file_path.relativePath,
-						pdf_header_template,
-						html_h1,
-					);
-			} else {
-				if (!inline_content)
-					doc_header = process_doc_header(
-						fm_headers,
-						file_path.relativePath,
-						doc_header_template_non_git,
-						html_h1,
-					);
-
-				if (pdf_enable && !pdf_path_excluded(file_path.relativePath))
-					pdf_header = process_doc_header(
-						fm_headers,
-						file_path.relativePath,
-						pdf_header_template_non_git,
-						html_h1,
-					);
-			}
-
-			let pdf_size = 0;
-			if (pdf_enable && !pdf_path_excluded(file_path.relativePath)) {
-				let pdf_txt = await hdoc_build_pdf.process_images(file_path, html_txt);
-				pdf_txt = `${pdf_header}\n${pdf_txt}`;
-
-				// Generate PDF file from HTML
-				const pdf_file_path = file_path.path.replace(
-					path.extname(file_path.path),
-					".pdf",
-				);
-				pdf_size = await hdoc_build_pdf.generate_pdf(
-					browser,
-					pdf_template_path,
-					pdf_template,
-					hdocbook_config,
-					pdf_txt,
-					pdf_file_path,
-					css_templates,
-					verbose,
-				);
-			}
-			if (pdf_size > 0) pdf_created++;
-
-			// Wrap h2 and h3 tags, plus content, in id'd divs
-			html_txt = hdoc.wrapHContent(html_txt);
-
-			if (inline_content) html_txt = `${fm_header_content}\n${html_txt}`;
-			else html_txt = `${fm_header_content}\n${doc_header}\n${html_txt}`;
-
-			let relative_path = file_path.relativePath;
-			if (
-				!bc[relative_path.replace(".html", "")] &&
-				bc[relative_path.replace("/index.html", "")]
-			) {
-				relative_path = relative_path.replace("/index.html", "");
-			}
-
-			const index_data = hdoc_index.transform_html_for_index(html_txt);
-
-			for (const section of index_data.sections) {
-				index_records.push({
-					relative_path: relative_path,
-					index_html: {
-						fm_props: index_data.fm_props,
-						text: section.text,
-						preview: section.preview,
-						id: section.id ? section.id : null,
-					},
-					metadata: metadata,
-					contributors: contribs,
-					pdf_size: pdf_size,
-					md5: file_path.hash,
-					lastmod: last_commit !== null ? last_commit : file_path.hb_lastmod,
-					inline: inline_content,
-					status: index_data.fm_props.status ? index_data.fm_props.status : 'release',
-					keywords: index_data.fm_props.keywords ? index_data.fm_props.keywords : '',
-				});
-			}
-
-			// Save HTML into HTML file
-			try {
-				fs.writeFileSync(file_path.path, html_txt);
-			} catch (err) {
-				console.error("Error writing:", target_file, "\n", err);
-			}
 		}
-	};
-
-	const transform_markdown_and_save_html = async (file_path) => {
-		conversion_attempted++;
-
-		if (fs.existsSync(file_path.path)) {
-			// Load markdown file
-			let md_txt = hdoc.expand_variables(
-				fs.readFileSync(file_path.path, "utf8"),
-			);
 
-			// Pull in external includes
-			const includes_processed = await hdoc.process_includes(
-				file_path.path,
-				md_txt,
-				global_source_path,
-			);
-			md_txt = includes_processed.body.toString();
-			includes_found += includes_processed.found;
-			includes_success += includes_processed.success;
-			includes_failed += includes_processed.failed;
-			if (includes_processed.errors.length > 0) {
-				for (let i = 0; i < includes_processed.errors.length; i++) {
-					console.error(includes_processed.errors[i]);
-				}
-			}
+		// ── Shared post-processing ─────────────────────────────────────────────
 
-			// One markdown parser per file. Seems wrong, but doesn't work with a global one once past the first md file
-			// Steve - revisit this
-			const md = require("markdown-it")({
-				html: true,
-				linkify: true,
-				typographer: true,
-				highlight: function (str, lang) {
-					
-					if (lang === "mermaid" && process.env.GITHUB_ACTIONS !== 'true') {
-						try {
-							const tmpInput = tmp.fileSync({ postfix: ".mmd" });
-							const outputFileName = `mermaid-${crypto.createHash("sha256").update(str).digest("hex").slice(0, 16)}.svg`;
-							const outputPath = path.join(mermaid_images_path, outputFileName);
-							const outputLink = `/_books/${doc_id}/mermaid-images/${outputFileName}`
-
-							if (!str.startsWith('---')) {
-								str = '---\n' + fs.readFileSync(mermaid_theme_path, {encoding: 'utf-8'}) + `\n---\n${str}`;
-							}
-
-							fs.writeFileSync(tmpInput.name, str);
-							let cmd = `${__dirname}/node_modules/.bin/mmdc`;
-
-							if (process.platform === "win32") {
-								cmd = `"${cmd}.cmd"`;
-							} 
-
-
-							cmd = `${cmd} -i "${tmpInput.name}" -o "${outputPath}" --backgroundColor transparent --puppeteerConfigFile ${mermaid_puppeteer_config_path}`;
-							console.log(`Generating Mermaid SVG found in ${file_path.relativePath} - ${outputPath}`);
-							execSync(cmd);
-							
-							if (!fs.existsSync(outputPath)) {
-								throw new Error("mmdc did not generate output");
-							}
-
-							tmpInput.removeCallback();
-
-							return `<img class="mermaid-diagram" src="${outputLink}" alt="Mermaid Diagram">`;
-						} catch (err) {
-							mermaid_failures.push({path: file_path.relativePath, error: err.message});
-							return  ``;
-						}
-					}
-				}
-			});
-			md.linkify.set({
-				fuzzyEmail: false,
-				fuzzyLink: false,
-				fuzzyIP: false,
-			});
+		let rel_path = file_path.relativePath.replace(path.extname(file_path.relativePath), '');
+		if (rel_path.endsWith('/index')) rel_path = rel_path.substring(0, rel_path.length - 6);
+		const is_draft = draft_links.indexOf(rel_path) !== -1;
 
-			// Process Frontmatter tags
-			let frontmatter_content = "";
-			md.use(mdfm, (fm) => {
-				frontmatter_content = fm;
-			});
+		// Add doc type
+		fm_headers.push({
+			id: "type",
+			value: doc_type,
+		});
 
-			// Process tips
-			const tips = require(`${__dirname}/custom_modules/tips.js`);
-			md.use(tips, {
-				links: true,
+		// Add status if we don't have one defined and the nav tree defines the document as draft
+		if (!fm_status && is_draft) {
+			fm_headers.push({
+				id: "status",
+				value: "draft"
 			});
+		}
 
-			// Tidy up ```json and ```xml code tags
-			if (md_txt.includes("```json") || md_txt.includes("```xml"))
-				md_txt = tidy_code_tags(md_txt, file_path.relativePath);
-
-			// Render markdown into HTML
-			let html_txt = md.render(md_txt);
-
-			// Prepare frontmatter headers
-			const fm_headers = [];
-			const fm_content = frontmatter_content.split(/\r?\n/);
-
-			let fm_contains_title = false;
-			let fm_contains_reading_time = false;
-			let fm_contains_description = false;
-			let doc_title = "";
-			let doc_type = "Article";
-
-			let rel_path = file_path.relativePath.replace(path.extname(file_path.relativePath), '');
-			if (rel_path.endsWith('/index')) rel_path = rel_path.substring(0, rel_path.length - 6);
-			const is_draft = draft_links.indexOf(rel_path) !== -1;
-			let fm_status = false;
+		const metadata = {};
 
-			if (fm_content.length >= 0) {
-				for (fm_prop of fm_content) {
-					const fm_id = fm_prop.slice(0, fm_prop.indexOf(":"));
-					const fm_val = fm_prop.slice(fm_prop.indexOf(":") + 1);
+		// Remove the first <h1>title</h1> from the HTML as we'll add that in the document header
+		let html_h1 = h1_pattern.exec(html_txt);
+		if (html_h1?.[0])
+			html_h1 = html_h1[0].replace(/(<h1.*?>)/, "").replace(/(<\/h1>)/, "");
 
-					if (
-						fm_id &&
-						fm_id.trim().length > 0 &&
-						fm_val &&
-						fm_val.trim().length > 0
-					) {
-						fm_headers.push({
-							id: fm_id.trim(),
-							value: fm_val.trim(),
-						});
+		html_txt = html_txt.replace(h1_pattern, "");
 
-						if (fm_id.trim() === "title") {
-							fm_contains_title = true;
-							doc_title = fm_val.trim();
-						}
-						if (fm_id.trim() === "status") {
-							fm_status = true;
-						}
-						if (fm_id.trim() === "type") {
-							doc_type = fm_val.trim();
-						}
-						if (fm_id.trim() === "reading-time") {
-							book_read_time += Number.parseInt(fm_val.trim(), 10);
-							fm_contains_reading_time = true;
-						}
-						if (fm_id.trim() === "description") {
-							fm_contains_description = true;
-						}
-					}
-				}
-			}
+		// Get contributor data from Github, if exists
+		let contribs = [];
+		let last_commit = null;
+		if (
+			hdocbook_config.publicSource &&
+			hdocbook_config.publicSource !== "" &&
+			hdocbook_config.publicSource.includes("github.com/Hornbill-Docs")
+		) {
+			const github_paths = hdoc.get_github_api_path(
+				hdocbook_config.publicSource,
+				file_path.relativePath,
+			);
+			const contributors = await hdoc.get_github_contributors(
+				github_paths.api_path,
+				git_token,
+				is_markdown ? hdocbook_config.publicSource : undefined,
+			);
 
-			// Add doc type
-			fm_headers.push({
-				id: "type",
-				value: doc_type,
-			});
+			if (!contributors.success) {
+				console.error(
+					`Error retrieving contributors from Github: ${contributors.error}`,
+				);
+			} else {
+				last_commit = contributors.last_commit_date;
+				metadata.last_commit = contributors.last_commit_date;
+				metadata.contributor_count = contributors.contributor_count;
+				metadata.edit_url = github_paths.edit_path;
+				contribs = contributors.contributors;
+				contributors.editPath = github_paths.edit_path;
 
-			// Add status if we don't have one defined and the nav tree defines the document as draft
-			if (!fm_status && is_draft) {
 				fm_headers.push({
-					id: "status",
-					value: "draft"
+					id: "contributor-count",
+					value: contributors.contributor_count,
 				});
-			}
-
-			// Does frontmatter tag contain a title property
-			if (!fm_contains_title) {
-				// Frontmatter tags don't contain a title property - go pull the first one from the html heading tags
-				const html_heading = hdoc.getFirstHTMLHeading(
-					html_txt,
-					h_tags_to_search,
-				);
-
-				if (html_heading?.[0]?.children?.[0]?.data) {
-					// We've found a heading tag, add that as a title to the frontmatter content
-					fm_headers.push({
-						id: "title",
-						value: html_heading[0].children[0].data.trim(),
-					});
-					doc_title = html_heading[0].children[0].data.trim();
-				} else if (
-					file_path.name !== "description_ext.md" &&
-					file_path.name !== "article_ext.md"
-				) {
-					// No header tag, no frontmatter title, output a warning
-					console.info(
-						`[WARNING] No frontmatter title property, or h1, h2 or h3 header tags detected in ${file_path.path}`,
-					);
-				}
-			}
-
-			// Does frontmatter contain a description header, generate one if not
-			if (!fm_contains_description) {
-				const html_p_tag = hdoc.getFirstHTMLHeading(html_txt, ["p"]);
-				if (html_p_tag?.[0]?.children?.[0]?.data) {
-					fm_headers.push({
-						id: "description",
-						value:
-							`${doc_title}: ${html_p_tag[0].children[0].data.split(".")[0]}.`.trim(),
-					});
-				}
-			}
-
-			// Does frontmatter tag contain a reading-time property
-			if (!fm_contains_reading_time) {
-				const read_time_mins = hdoc.get_html_read_time(html_txt);
-				book_read_time += read_time_mins;
 				fm_headers.push({
-					id: "reading-time",
-					value: read_time_mins,
+					id: "last-commit",
+					value: contributors.last_commit_date,
 				});
-			}
-			const metadata = {};
-
-			// Remove the first <h1>title</h1> from the HTML as we'll add that in the document header
-			let html_h1 = h1_pattern.exec(html_txt);
-			if (html_h1?.[0])
-				html_h1 = html_h1[0].replace(/(<h1.*?>)/, "").replace(/(<\/h1>)/, "");
-
-			html_txt = html_txt.replace(h1_pattern, "");
-
-			// Get contributor data from Github, if exists
-			let contribs = [];
-			let last_commit = null;
-			if (
-				hdocbook_config.publicSource &&
-				hdocbook_config.publicSource !== "" &&
-				hdocbook_config.publicSource.includes("github.com/Hornbill-Docs")
-			) {
-				const github_paths = hdoc.get_github_api_path(
-					hdocbook_config.publicSource,
-					file_path.relativePath,
-				);
-				const contributors = await hdoc.get_github_contributors(
-					github_paths.api_path,
-					git_token,
-					hdocbook_config.publicSource,
-				);
+				contributors.success = undefined;
+				contributors.error = undefined;
+				contributors.editPath = github_paths.edit_path;
 
-				if (!contributors.success) {
-					console.error(
-						`Error retrieving contributors from Github: ${contributors.error}`,
-					);
-				} else {
-					last_commit = contributors.last_commit_date;
-					metadata.last_commit = contributors.last_commit_date;
-					metadata.contributor_count = contributors.contributor_count;
-					metadata.edit_url = github_paths.edit_path;
-					contribs = contributors.contributors;
-					contributors.editPath = github_paths.edit_path;
-
-					fm_headers.push({
-						id: "contributor-count",
-						value: contributors.contributor_count,
-					});
-					fm_headers.push({
-						id: "last-commit",
-						value: contributors.last_commit_date,
-					});
-					const target_file = file_path.path.replace(
+				if (is_markdown) {
+					// Write contributor metadata to a sidecar JSON file for later use
+					const target_info_file = file_path.path.replace(
 						path.extname(file_path.path),
 						"._info.json",
 					);
-					contributors.success = undefined;
-					contributors.error = undefined;
-					contributors.editPath = github_paths.edit_path;
 					try {
 						fs.writeFileSync(
-							target_file,
+							target_info_file,
 							JSON.stringify(contributors, null, 2),
 						);
 					} catch (err) {
-						console.error("Error writing:", target_file, "\n", err);
+						console.error("Error writing:", target_info_file, "\n", err);
 					}
 				}
-				fm_headers.push({
-					id: "edit-path",
-					value: github_paths.edit_path,
-				});
 			}
+			fm_headers.push({
+				id: "edit-path",
+				value: github_paths.edit_path,
+			});
+		}
 
-			if (pdf_enable && !pdf_path_excluded(file_path.relativePath)) {
-				fm_headers.push({
-					id: "pdf-path",
-					value: file_path.relativePath.replace(
-						path.extname(file_path.relativePath),
-						".pdf",
-					),
-				});
-			}
+		if (pdf_enable && !pdf_path_excluded(file_path.relativePath)) {
+			fm_headers.push({
+				id: "pdf-path",
+				value: file_path.relativePath.replace(
+					path.extname(file_path.relativePath),
+					".pdf",
+				),
+			});
+		}
 
-			// Add frontmatter tags as comment
-			let fm_header = "<!--[[FRONTMATTER\n";
-			if (fm_headers.length > 0) {
-				for (let i = 0; i < fm_headers.length; i++) {
-					fm_header += `${fm_headers[i].id}: ${fm_headers[i].value}\n`;
-				}
+		// Build frontmatter comment block
+		let fm_header_str = "<!--[[FRONTMATTER\n";
+		if (fm_headers.length > 0) {
+			for (let i = 0; i < fm_headers.length; i++) {
+				fm_header_str += `${fm_headers[i].id}: ${fm_headers[i].value}\n`;
 			}
-			fm_header += "]]-->";
-
-			let doc_header = "";
-			let pdf_header = "";
-			const inline_content = file_path.relativePath.startsWith(
-				`${hdocbook_config.docId}/_inline/`,
-			);
-			if (hdocbook_config.publicSource?.includes("github.com/Hornbill-Docs")) {
-				// Build doc header from template and frontmatter tags
-				if (!inline_content)
-					doc_header = process_doc_header(
-						fm_headers,
-						file_path.relativePath,
-						doc_header_template,
-						html_h1,
-					);
-
-				if (pdf_enable && !pdf_path_excluded(file_path.relativePath))
-					pdf_header = process_doc_header(
-						fm_headers,
-						file_path.relativePath,
-						pdf_header_template,
-						html_h1,
-					);
-			} else {
-				// Build doc header from template and frontmatter tags
-				if (!inline_content)
-					doc_header = process_doc_header(
-						fm_headers,
-						file_path.relativePath,
-						doc_header_template_non_git,
-						html_h1,
-					);
-
-				if (pdf_enable && !pdf_path_excluded(file_path.relativePath))
-					pdf_header = process_doc_header(
-						fm_headers,
-						file_path.relativePath,
-						pdf_header_template_non_git,
-						html_h1,
-					);
+			// For static HTML, strip the original frontmatter comment before prepending the new one
+			if (!is_markdown && existing_fm_headers) {
+				html_txt = html_txt.replace(`<!--${html_fm.fm_header}-->`, "");
 			}
+		}
+		fm_header_str += "]]-->";
 
-			let pdf_size = 0;
-			if (pdf_enable && !pdf_path_excluded(file_path.relativePath)) {
-				let pdf_txt = await hdoc_build_pdf.process_images(file_path, html_txt);
-				pdf_txt = `${pdf_header}\n${pdf_txt}`;
+		let doc_header = "";
+		let pdf_header = "";
+		const inline_content = file_path.relativePath.startsWith(
+			`${hdocbook_config.docId}/_inline/`,
+		);
+		if (hdocbook_config.publicSource?.includes("github.com/Hornbill-Docs")) {
+			// Build doc header from template and frontmatter tags
+			if (!inline_content)
+				doc_header = process_doc_header(
+					fm_headers,
+					file_path.relativePath,
+					doc_header_template,
+					html_h1,
+				);
 
-				// Generate PDF file from HTML
-				const pdf_file_path = file_path.path.replace(
-					path.extname(file_path.path),
-					".pdf",
+			if (pdf_enable && !pdf_path_excluded(file_path.relativePath))
+				pdf_header = process_doc_header(
+					fm_headers,
+					file_path.relativePath,
+					pdf_header_template,
+					html_h1,
 				);
-				pdf_size = await hdoc_build_pdf.generate_pdf(
-					browser,
-					pdf_template_path,
-					pdf_template,
-					hdocbook_config,
-					pdf_txt,
-					pdf_file_path,
-					css_templates,
-					verbose,
+		} else {
+			if (!inline_content)
+				doc_header = process_doc_header(
+					fm_headers,
+					file_path.relativePath,
+					doc_header_template_non_git,
+					html_h1,
 				);
-			}
-			if (pdf_size > 0) pdf_created++;
 
-			// Wrap h2 and h3 tags, plus content, in id'd divs
-			html_txt = hdoc.wrapHContent(html_txt);
+			if (pdf_enable && !pdf_path_excluded(file_path.relativePath))
+				pdf_header = process_doc_header(
+					fm_headers,
+					file_path.relativePath,
+					pdf_header_template_non_git,
+					html_h1,
+				);
+		}
 
-			if (inline_content) html_txt = `${fm_header}\n${html_txt}`;
-			else html_txt = `${fm_header}\n${doc_header}\n${html_txt}`;
+		let pdf_size = 0;
+		if (pdf_enable && !pdf_path_excluded(file_path.relativePath)) {
+			let pdf_txt = await hdoc_build_pdf.process_images(file_path, html_txt);
+			pdf_txt = `${pdf_header}\n${pdf_txt}`;
 
-			// Save HTML into HTML file
-			const target_file = file_path.path.replace(
+			// Generate PDF file from HTML
+			const pdf_file_path = file_path.path.replace(
 				path.extname(file_path.path),
-				".html",
+				".pdf",
 			);
-			let relative_path = file_path.relativePath.replace(
-				path.extname(file_path.path),
-				".html",
+			pdf_size = await hdoc_build_pdf.generate_pdf(
+				browser,
+				pdf_template_path,
+				pdf_template,
+				hdocbook_config,
+				pdf_txt,
+				pdf_file_path,
+				css_templates,
+				verbose,
 			);
-			try {
-				fs.writeFileSync(target_file, html_txt);
-			} catch (err) {
-				console.error("Error writing:", target_file, "\n", err);
-			}
+		}
+		if (pdf_size > 0) pdf_created++;
 
-			if (
-				!bc[relative_path.replace(".html", "")] &&
-				bc[relative_path.replace("/index.html", "")]
-			) {
-				relative_path = relative_path.replace("/index.html", "");
-			}
+		// Wrap h2 and h3 tags, plus content, in id'd divs
+		html_txt = hdoc.wrapHContent(html_txt);
 
-			const index_data = hdoc_index.transform_html_for_index(html_txt);
-			for (section of index_data.sections) {
-				index_records.push({
-					relative_path: relative_path,
-					index_html: {
-						fm_props: index_data.fm_props,
-						text: section.text,
-						preview: section.preview,
-						id: section.id ? section.id : null,
-					},
-					metadata: metadata,
-					contributors: contribs,
-					pdf_size: pdf_size,
-					md5: file_path.hash,
-					lastmod: last_commit !== null ? last_commit : file_path.hb_lastmod,
-					inline: inline_content,
-					status: index_data.fm_props.status ? index_data.fm_props.status : 'release',
-					keywords: index_data.fm_props.keywords ? index_data.fm_props.keywords : '',
-				});
-			}
+		if (inline_content) html_txt = `${fm_header_str}\n${html_txt}`;
+		else html_txt = `${fm_header_str}\n${doc_header}\n${html_txt}`;
 
+		// Determine output file path (.md → .html for markdown; same path for static HTML)
+		const target_file = is_markdown
+			? file_path.path.replace(path.extname(file_path.path), ".html")
+			: file_path.path;
+		try {
+			fs.writeFileSync(target_file, html_txt);
+		} catch (err) {
+			console.error("Error writing:", target_file, "\n", err);
+		}
+
+		let relative_path = is_markdown
+			? file_path.relativePath.replace(path.extname(file_path.path), ".html")
+			: file_path.relativePath;
+		if (
+			!bc[relative_path.replace(".html", "")] &&
+			bc[relative_path.replace("/index.html", "")]
+		) {
+			relative_path = relative_path.replace("/index.html", "");
+		}
+
+		const index_data = hdoc_index.transform_html_for_index(html_txt);
+		for (const section of index_data.sections) {
+			index_records.push({
+				relative_path: relative_path,
+				index_html: {
+					fm_props: index_data.fm_props,
+					text: section.text,
+					preview: section.preview,
+					id: section.id ? section.id : null,
+				},
+				metadata: metadata,
+				contributors: contribs,
+				pdf_size: pdf_size,
+				md5: file_path.hash,
+				lastmod: last_commit !== null ? last_commit : file_path.hb_lastmod,
+				inline: inline_content,
+				status: index_data.fm_props.status ? index_data.fm_props.status : 'release',
+				keywords: index_data.fm_props.keywords ? index_data.fm_props.keywords : '',
+			});
+		}
+
+		if (is_markdown) {
 			// Add MD file to delete queue
 			md_files_delete.push(file_path.path);
-
 			conversion_success++;
 			return true;
 		}
-		conversion_failed++;
-		console.error("MD file does not exist:", file_path.path);
-		return false;
 	};
 
 	const tidy_code_tags = (markdown, file) => {
@@ -950,7 +763,7 @@
 						.replace("```", "");
 					let new_xml_string = xml_tidy;
 					try {
-						new_xml_string = xmlFormat(xml_tidy, {
+						new_xml_string = hdoc.xml_format(xml_tidy, {
 							indentation: "  ",
 							collapseContent: true,
 							lineSeparator: "\n",
@@ -1105,7 +918,7 @@
 	const filename_validation_callback = (element) => {
 		if (element.relativePath.startsWith("_inline/")) return;
 		if (element.name.toLowerCase() === ".ds_store") return;
-		if (element.name === "article_ext.md" || element.name === "description_ext.md" ) return;
+		if (element.name === "article_ext.md" || element.name === "description_ext.md" || element.name === "internal_ext.md" ) return;
 
 		const file_no_ext = element.name.replace(`.${element.extension}`, "");
 		
@@ -1290,7 +1103,7 @@
 
 		// Validate all filenames first
 		console.log("Validating book filenames meet kebab-case requirements...");
-		dree.scan(book_path, dreeOptionsAllFiles, filename_validation_callback);
+		hdoc.scan_dir(book_path, dreeOptionsAllFiles, filename_validation_callback);
 		if (errors_filename.length > 0) {
 			console.log("\r\n-----------------------");
 			console.log("   Validation Output   ");
@@ -1322,7 +1135,8 @@
 		// Copy files from book into _work-doc_id folder
 		console.log("Copying content into work folder...");
 		try {
-			fs.copySync(path.join(source_path, doc_id), work_path_content, {
+			fs.cpSync(path.join(source_path, doc_id), work_path_content, {
+				recursive: true,
 				filter: file_filter,
 			});
 		} catch (e) {
@@ -1336,7 +1150,7 @@
 		// Create MD5 hash of content before build
 		console.log("Creating Hash...");
 
-		dree.scan(work_path_content, md5DreeOptions, hash_callback);
+		hdoc.scan_dir(work_path_content, md5DreeOptions, hash_callback);
 		let concat_hash = "|";
 		for (let i = 0; i < built_file_hashes.length; i++) {
 			concat_hash += `${built_file_hashes[i].path}:${built_file_hashes[i].hash}|`;
@@ -1405,7 +1219,7 @@
 		console.log("Processing content...");
 
 		// Get a list of MD files in work_path
-		dree.scan(work_path, dreeOptions, build_file_callback);
+		hdoc.scan_dir(work_path, dreeOptions, build_file_callback);
 
 		// Create a Chromium browser instance generate PDFs and validate links with
 		browser = await puppeteer.launch({ headless: "shell", args: ['--no-sandbox'] });
@@ -1421,7 +1235,7 @@
 			// do whatever
 			await Promise.all(
 				chunk.map(async (file) => {
-					await transform_markdown_and_save_html(file);
+					await transform_file(file);
 				}),
 			);
 		}
@@ -1435,7 +1249,7 @@
 			const chunk = htmlPromiseArray.slice(i, i + chunkSize);
 			await Promise.all(
 				chunk.map(async (file) => {
-					await transform_static_html(file);
+					await transform_file(file);
 				}),
 			);
 		}