hdoc-tools 0.31.0 → 0.32.1

package/hdoc-build-pdf.js

@@ -1,219 +1,233 @@
-(() => {
-	const axios = require("axios");
-	const cheerio = require("cheerio");
-	const fs = require("fs-extra");
-	const mime = require("mime-types");
-	const path = require("node:path");
-	const hdoc = require(path.join(__dirname, "hdoc-module.js"));
-
-	let hb_logo = "";
-	let footer = "";
-	let header = "";
-
-	const get_footer = (template_path) => {
-		let footer_content = null;
-		try {
-			footer_content = fs.readFileSync(
-				path.join(template_path, "template-footer.html"),
-				"utf8",
-			);
-		} catch (err) {
-			console.error(`Error loading template: ${err}`);
-		}
-		return footer_content;
-	};
-
-	const get_header = (template_path) => {
-		let header_content = null;
-		try {
-			header_content = fs.readFileSync(
-				path.join(template_path, "template-header.html"),
-				"utf8",
-			);
-		} catch (err) {
-			console.error(`Error loading template: ${err}`);
-		}
-		return header_content;
-	};
-
-	exports.process_images = async (file_path, html_source, verbose) => {
-		const book_work_root = file_path.path.replace(file_path.relativePath, "");
-		if (verbose) console.log("Parsing img tags from HTML source");
-
-		let processed_html_source = html_source;
-		// Use cheerio to parse html
-		const $ = cheerio.load(processed_html_source);
-
-		// Get iFrames from HTML, to replace with a tags
-		const iframes = [];
-		const iframe_html = $("iframe")
-			.map(function () {
-				const response = {
-					html: $.html(this),
-					src: $(this).attr("src"),
-					title: $(this).attr("title")
-						? $(this).attr("title")
-						: "No Link Title Provided",
-				};
-				return response;
-			})
-			.get();
-		iframes.push(...iframe_html);
-		for (let i = 0; i < iframes.length; i++) {
-			const link = `<p><a href="${iframes[i].src}">${iframes[i].title}</a></p>`;
-			const regex = new RegExp(
-				`<iframe.*src="${iframes[i].src.replace("/", "\\/")}".*</iframe>`,
-			);
-			processed_html_source = processed_html_source.replace(regex, link);
-		}
-
-		// Get image links from HTML, to embed into the pdf
-		const imgs = [];
-		const srcs = $("img")
-			.map(function (i) {
-				return $(this).attr("src");
-			})
-			.get();
-		imgs.push(...srcs);
-		for (let i = 0; i < imgs.length; i++) {
-			if (!hdoc.valid_url(imgs[i])) {
-				// Internal link
-				const image_path = path.join(
-					book_work_root,
-					imgs[i].replace("_books/", ""),
-				);
-				try {
-					const image_buffer = fs.readFileSync(image_path);
-					const mime_type = mime.lookup(image_path);
-					let image_b64 = image_buffer.toString("base64");
-					image_b64 = `data:${mime_type};base64,${image_b64}`;
-					processed_html_source = processed_html_source.replace(
-						imgs[i],
-						image_b64,
-					);
-				} catch (err) {
-					console.error(
-						"Error reading image from HTML source [",
-						image_path,
-						"] -",
-						err,
-					);
-					return null;
-				}
-			} else {
-				// External Link
-				try {
-					const file_response = await axios.get(imgs[i]);
-					if (file_response.status === 200) {
-						const image_buffer = file_response.data;
-						const mime_type = mime.lookup(imgs[i]);
-						let image_b64 = image_buffer.toString("base64");
-						image_b64 = `data:${mime_type};base64,${image_b64}`;
-						processed_html_source = processed_html_source.replace(
-							imgs[i],
-							image_b64,
-						);
-					} else {
-						throw `Unexpected Status ${file_response.status}`;
-					}
-				} catch (err) {
-					console.error(
-						`Error downloading external source [${imgs[i]}] - ${err}`,
-					);
-				}
-			}
-		}
-
-		return processed_html_source;
-	};
-
-	exports.generate_pdf = async (
-		browser,
-		pdf_template_path,
-		pdf_template_content,
-		book_config,
-		html_source,
-		target_file,
-		css_templates,
-		verbose = false,
-	) => {
-		let pdf_size = 0;
-		// Cache footer
-		if (footer === "") footer = get_footer(pdf_template_path);
-
-		// Read svg logo file into buffer, convert to B64 string
-		if (hb_logo === "") {
-			const hb_logo_path = path.join(
-				pdf_template_path,
-				"images",
-				"hornbill-logo-full.svg",
-			);
-			try {
-				const hb_logo_file_buffer = fs.readFileSync(hb_logo_path);
-				hb_logo = hb_logo_file_buffer.toString("base64");
-				hb_logo = `data:image/svg+xml;base64,${hb_logo}`;
-			} catch (err) {
-				console.error("Error reading logo from template:", err);
-				return pdf_size;
-			}
-		}
-
-		// Cache header
-		if (header === "") {
-			header = get_header(pdf_template_path)
-				.replace("{{book_title}}", book_config.title)
-				.replace("{{hb_logo}}", hb_logo);
-		}
-
-		const processed_html_source = pdf_template_content
-			.replace("{{book_title}}", book_config.title)
-			.replace("{{document_content}}", html_source);
-
-		const page = await browser.newPage();
-
-		// To reflect CSS used for screens instead of print
-		await page.emulateMediaType("screen");
-
-		// Set HTML content from HTML source
-		await page.setContent(processed_html_source, {
-			waitUntil: "domcontentloaded",
-		});
-		for (let i = 0; i < css_templates.length; i++) {
-			try {
-				await page.addStyleTag({
-					content: css_templates[i],
-				});
-			} catch (e) {
-				console.error(`Error applying template for [${target_file}]: ${e}`);
-			}
-		}
-
-		try {
-			const pdf_gen = await page.pdf({
-				path: target_file,
-				printBackground: true,
-				format: "A4",
-				displayHeaderFooter: true,
-				headerTemplate: header,
-				footerTemplate: footer,
-				margin: {
-					top: "90px",
-					right: "30px",
-					bottom: "60px",
-					left: "30px",
-				},
-				timeout: 0,
-			});
-			const currdate = new Date();
-			const datetime = currdate.toISOString();
-			if (verbose)
-				console.log(`[${datetime}] PDF generation success: ${target_file}`);
-
-			pdf_size = pdf_gen.byteLength;
-		} catch (err) {
-			console.error(`Error generating PDF ${target_file} - ${err}`);
-		}
-		await page.close();
-		return pdf_size;
-	};
-})();
+(() => {
+	const axios = require("axios");
+	const cheerio = require("cheerio");
+	const fs = require("fs-extra");
+	const mime = require("mime-types");
+	const path = require("node:path");
+	const hdoc = require(path.join(__dirname, "hdoc-module.js"));
+
+	let hb_logo = "";
+	let footer = "";
+	let header = "";
+
+	const get_footer = (template_path) => {
+		let footer_content = null;
+		try {
+			footer_content = fs.readFileSync(
+				path.join(template_path, "template-footer.html"),
+				"utf8",
+			);
+		} catch (err) {
+			console.error(`Error loading template: ${err}`);
+		}
+		return footer_content;
+	};
+
+	const get_header = (template_path) => {
+		let header_content = null;
+		try {
+			header_content = fs.readFileSync(
+				path.join(template_path, "template-header.html"),
+				"utf8",
+			);
+		} catch (err) {
+			console.error(`Error loading template: ${err}`);
+		}
+		return header_content;
+	};
+
+	exports.process_images = async (file_path, html_source, verbose) => {
+		const book_work_root = file_path.path.replace(file_path.relativePath, "");
+		if (verbose) console.log("Parsing img tags from HTML source");
+
+		let processed_html_source = html_source;
+		// Use cheerio to parse html
+		const $ = cheerio.load(processed_html_source);
+
+		// Get iFrames from HTML, to replace with a tags
+		const iframes = [];
+		const iframe_html = $("iframe")
+			.map(function () {
+				const response = {
+					html: $.html(this),
+					src: $(this).attr("src"),
+					title: $(this).attr("title")
+						? $(this).attr("title")
+						: "No Link Title Provided",
+				};
+				return response;
+			})
+			.get();
+		iframes.push(...iframe_html);
+		for (let i = 0; i < iframes.length; i++) {
+			const link = `<p><a href="${iframes[i].src}">${iframes[i].title}</a></p>`;
+			const regex = new RegExp(
+				`<iframe.*src="${iframes[i].src.replace("/", "\\/")}".*</iframe>`,
+			);
+			processed_html_source = processed_html_source.replace(regex, link);
+		}
+
+		// Get image links from HTML, to embed into the pdf
+		const imgs = [];
+		const srcs = $("img")
+			.map(function (i) {
+				return $(this).attr("src");
+			})
+			.get();
+		imgs.push(...srcs);
+		for (let i = 0; i < imgs.length; i++) {
+			if (!hdoc.valid_url(imgs[i])) {
+				// Internal link
+				const image_path = path.join(
+					book_work_root,
+					imgs[i].replace("_books/", ""),
+				);
+				try {
+					const image_buffer = fs.readFileSync(image_path);
+					const mime_type = mime.lookup(image_path);
+					let image_b64 = image_buffer.toString("base64");
+					image_b64 = `data:${mime_type};base64,${image_b64}`;
+					processed_html_source = processed_html_source.replace(
+						imgs[i],
+						image_b64,
+					);
+				} catch (err) {
+					console.error(
+						"Error reading image from HTML source [",
+						image_path,
+						"] -",
+						err,
+					);
+					return null;
+				}
+			} else {
+				// External Link
+				try {
+					const file_response = await axios.get(imgs[i], {
+						responseType: 'arraybuffer' 
+					});
+					if (file_response.status === 200) {
+						let image_b64 = imageEncode(file_response.data, file_response.headers['content-type']);
+						
+						
+						const regexQ = `<img\\s+[^>]*src=["']${imgs[i].replaceAll('/', '\\/').replaceAll('.', '\\.')}["'][^>]*>`;
+						const regex = new RegExp(regexQ);
+
+						const found_img_tag = processed_html_source.match(regex);
+						const new_img_tag = found_img_tag[0].replace(imgs[i], image_b64);
+						
+						processed_html_source = processed_html_source.replace(
+							found_img_tag,
+							new_img_tag,
+						);
+					} else {
+						throw `Unexpected Status ${file_response.status}`;
+					}
+				} catch (err) {
+					console.error(
+						`Error downloading external source [${imgs[i]}] - ${err}`,
+					);
+				}
+			}
+		}
+
+		return processed_html_source;
+	};
+
+	const imageEncode = (arrayBuffer, mimeType) => {
+		let u8 = new Uint8Array(arrayBuffer)
+		let b64encoded = btoa([].reduce.call(new Uint8Array(arrayBuffer),function(p,c){return p+String.fromCharCode(c)},''))
+		let mimetype=`image/${mimeType}`
+		return "data:"+mimetype+";base64,"+b64encoded
+	}
+
+	exports.generate_pdf = async (
+		browser,
+		pdf_template_path,
+		pdf_template_content,
+		book_config,
+		html_source,
+		target_file,
+		css_templates,
+		verbose = false,
+	) => {
+		let pdf_size = 0;
+		// Cache footer
+		if (footer === "") footer = get_footer(pdf_template_path);
+
+		// Read svg logo file into buffer, convert to B64 string
+		if (hb_logo === "") {
+			const hb_logo_path = path.join(
+				pdf_template_path,
+				"images",
+				"hornbill-logo-full.svg",
+			);
+			try {
+				const hb_logo_file_buffer = fs.readFileSync(hb_logo_path);
+				hb_logo = hb_logo_file_buffer.toString("base64");
+				hb_logo = `data:image/svg+xml;base64,${hb_logo}`;
+			} catch (err) {
+				console.error("Error reading logo from template:", err);
+				return pdf_size;
+			}
+		}
+
+		// Cache header
+		if (header === "") {
+			header = get_header(pdf_template_path)
+				.replace("{{book_title}}", book_config.title)
+				.replace("{{hb_logo}}", hb_logo);
+		}
+
+		const processed_html_source = pdf_template_content
+			.replace("{{book_title}}", book_config.title)
+			.replace("{{document_content}}", html_source);
+
+		const page = await browser.newPage();
+
+		// To reflect CSS used for screens instead of print
+		await page.emulateMediaType("screen");
+
+		// Set HTML content from HTML source
+		await page.setContent(processed_html_source, {
+			waitUntil: "domcontentloaded",
+		});
+		for (let i = 0; i < css_templates.length; i++) {
+			try {
+				await page.addStyleTag({
+					content: css_templates[i],
+				});
+			} catch (e) {
+				console.error(`Error applying template for [${target_file}]: ${e}`);
+			}
+		}
+
+		try {
+			const pdf_gen = await page.pdf({
+				path: target_file,
+				printBackground: true,
+				format: "A4",
+				displayHeaderFooter: true,
+				headerTemplate: header,
+				footerTemplate: footer,
+				margin: {
+					top: "90px",
+					right: "30px",
+					bottom: "60px",
+					left: "30px",
+				},
+				timeout: 0,
+			});
+			const currdate = new Date();
+			const datetime = currdate.toISOString();
+			if (verbose)
+				console.log(`[${datetime}] PDF generation success: ${target_file}`);
+
+			pdf_size = pdf_gen.byteLength;
+		} catch (err) {
+			console.error(`Error generating PDF ${target_file} - ${err}`);
+		}
+		await page.close();
+		return pdf_size;
+	};
+})();

package/hdoc-build.js

@@ -60,6 +60,7 @@
 	let conversion_failed = 0;
 	let doc_header_template = "";
 	let doc_header_template_non_git = "";
+	let github_repo_details = {};
 	let global_source_path = "";
 	let pdf_created = 0;
 	let pdf_enable = false;
@@ -306,7 +307,8 @@
 					hdocbook_config.publicSource,
 					file_path.relativePath,
 				);
-				const contributors = hdoc.get_github_contributors(
+				
+				const contributors = await hdoc.get_github_contributors(
 					github_paths.api_path,
 					git_token,
 				);
@@ -1184,6 +1186,18 @@
 			prods_supported = prods.prods_supported;
 		}
 
+		const clean_repo = hdocbook_config.publicSource.endsWith("/") ? hdocbook_config.publicSource.slice(0, -1) : hdocbook_config.publicSource;
+		const api_path = clean_repo.replace(
+			"https://github.com/",
+			"https://api.github.com/repos/",
+		);
+
+		// Get github repo details
+		github_repo_details = await hdoc.get_github_repo_details( api_path, git_token ); 
+		if (github_repo_details.success) {
+			console.warn(`Unable to retrieve GitHub Repository details: ${github_repo_details.error}`);
+		}
+		
 		if (!validate) {
 			console.log("Caching CSS for PDF generation...");
 			const css_files = [
@@ -1394,6 +1408,7 @@
 			gen_exclude,
 			redirects,
 			draft_links,
+			github_repo_details.data.private,
 		);
 		if (!validation_success) {
 			const end_time = Date.now();

package/hdoc-module.js

@@ -434,6 +434,60 @@
 		return github_paths;
 	};
 
+	exports.get_github_repo_details = async (
+		github_url,
+		github_api_token,
+	) => {
+		const response = {
+			success: false,
+			error: "",
+			data: {},
+			private: false
+		};
+		const request_options = {
+			headers: {
+				"User-Agent": "HornbillDocsBuild",
+				"Cache-Control": "no-cache",
+				Host: "api.github.com",
+				Accept: "application/json",
+			},
+			timeout: 5000,
+		};
+		if (github_api_token !== "") {
+			request_options.headers.authorization = `Bearer ${github_api_token}`;
+		}
+
+		let github_response;
+		try {
+			github_response = await axios.get(github_url, request_options);
+			if (retried) {
+				retried = false;
+				console.log("API call retry success!");
+			}
+		} catch (err) {
+			if (err.response) {
+				if (err.response.status !== 403 && err.response.status !== 401) {
+					response.error = err;
+					return response;
+				}
+				github_response = err.response;
+			} else {
+				response.error = `Unexpected response from GitHub for [${github_url}:\n${JSON.stringify(
+					err,
+				)}]`;
+			}
+		}
+		if (github_response.status === 200) {
+			response.success = true;
+			response.data = github_response.data;
+			response.private = github_response.data.private;
+		} else {
+			// Is it a 404 or 403?
+			response.error = `${github_response.status} : ${data.message}`;
+		}
+		return response;
+	};
+
 	exports.get_github_contributors = async (
 		github_url,
 		github_api_token,

package/hdoc-validate.js

@@ -28,6 +28,7 @@ const e = require("express");
 	const md_to_validate = [];
 	const exclude_links = {};
 	const exclude_spellcheck = {};
+	let private_repo = false;
 	let redirects = {};
 	const exclude_h1_count = {};
 	const exclude_spellcheck_output = [];
@@ -545,6 +546,17 @@ const e = require("express");
 					continue;
 				}
 
+				if ( 
+					links[i].toLowerCase().includes("docs-internal.hornbill.com") &&
+					markdown_paths.relativePath.includes('/_inline/') &&
+					!private_repo
+				) {
+					// Is the parent book in a public repo? If so, flag this as an error.
+					const error_message = processErrorMessage(`Hornbill docs-internal links should not be used in public book inline content: ${links[i]}`, markdown_paths.relativePath, markdown_content, links[i]);
+					errors[htmlFile.relativePath].push( error_message );
+					continue;
+				}
+
 				try {
 					await axios({
 						url: links[i],
@@ -810,9 +822,11 @@ const e = require("express");
 		gen_exclude,
 		gen_redirects,
 		draft_links,
+		is_private,
 	) => {
 		console.log("Performing Validation and Building SEO Link List...");
 		redirects = gen_redirects;
+		private_repo = is_private;
 
 		// Get a list of HTML files in source_path
 		dree.scan(source_path, dreeOptions, fileContentCallback);

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "hdoc-tools",
-	"version": "0.31.0",
+	"version": "0.32.1",
 	"description": "Hornbill HDocBook Development Support Tool",
 	"main": "hdoc.js",
 	"bin": {