hdoc-tools 0.34.2 → 0.35.0

package/hdoc-build.js

@@ -72,7 +72,7 @@
 	let prods_supported = [];
 	let doc_id = "";
 	let git_token =
-		"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"; // Github fine-grained personal access token that has minimum read-only access to Hornbill Docs metadata
+		"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"; // Github fine-grained personal access token that has minimum read-only access to Hornbill Docs metadata
 	let hdocbook_config = {};
 	let hdocbook_project;
 	let includes_found = 0;
@@ -1201,7 +1201,7 @@
 	
 			// Get github repo details
 			github_repo_details = await hdoc.get_github_repo_details( api_path, git_token ); 
-			if (github_repo_details.success) {
+			if (!github_repo_details.success) {
 				console.warn(`Unable to retrieve GitHub Repository details: ${github_repo_details.error}`);
 			}
 		}
@@ -1345,10 +1345,8 @@
 		// Get a list of MD files in work_path
 		dree.scan(work_path, dreeOptions, build_file_callback);
 
-		if (pdf_enable) {
-			// Create a Chromium browser instance generate PDFs with
-			browser = await puppeteer.launch({ headless: "shell" });
-		}
+		// Create a Chromium browser instance generate PDFs and validate links with
+		browser = await puppeteer.launch({ headless: "shell" });
 
 		// Work through MD files and convert to HTML
 		const mdPromiseArray = [];
@@ -1380,11 +1378,6 @@
 			);
 		}
 
-		if (pdf_enable) {
-			// Close the Chromium browser instance
-			await browser.close();
-		}
-
 		// Output to console
 		console.log(`\n                MD files found: ${conversion_attempted}`);
 		console.log(`Successfully converted to HTML: ${conversion_success}`);
@@ -1417,7 +1410,12 @@
 			redirects,
 			draft_links,
 			github_repo_details && github_repo_details.data && github_repo_details.data.private ? github_repo_details.data.private : false,
+			browser
 		);
+
+		// Close the Chromium browser instance
+		await browser.close();
+
 		if (!validation_success) {
 			const end_time = Date.now();
 			console.log(`\nTime Taken: ${get_duration(start_time, end_time)}\n`);

package/hdoc-module.js

@@ -483,7 +483,7 @@
 			response.private = github_response.data.private;
 		} else {
 			// Is it a 404 or 403?
-			response.error = `${github_response.status} : ${data.message}`;
+			response.error = `${github_response.status} : ${github_response.data.message}`;
 		}
 		return response;
 	};

package/hdoc-validate.js

@@ -7,7 +7,6 @@ const e = require("express");
 	const dns = require("node:dns");
 	const fs = require("node:fs");
 	const path = require("node:path");
-	const https = require("node:https");
 	const hdoc = require(path.join(__dirname, "hdoc-module.js"));
 	const translator = require("american-british-english-translator");
 	const { trueCasePathSync } = require("true-case-path");
@@ -17,9 +16,6 @@ const e = require("express");
 		spelling: true,
 	};
 	const regex_nav_paths = /[a-z0-9-\/]+[a-z0-9]+#{0,1}[a-z0-9-\/]+/;
-	const agent = new https.Agent({
-		rejectUnauthorized: false,
-	});
 
 	const errors = {};
 	const messages = {};
@@ -449,7 +445,7 @@ const e = require("express");
 		return returnPaths;
 	}
 
-	const checkLinks = async (source_path, htmlFile, links, hdocbook_config, hdocbook_project) => {
+	const checkLinks = async (source_path, htmlFile, links, hdocbook_config, hdocbook_project, browser) => {
 		const markdown_paths = getMDPathFromHtmlPath(htmlFile);
 		const markdown_content = fs.readFileSync(markdown_paths.markdownPath, 'utf8');
 
@@ -557,19 +553,62 @@ const e = require("express");
 					continue;
 				}
 
+
 				try {
-					await axios({
-						url: links[i],
-						method: 'get',
-						timeout: 30000,
-						maxRedirects: 5,
-						validateStatus: (status) =>
-						  status >= 200 && status < 400,
-					  })
 
-					messages[htmlFile.relativePath].push(
-						`Link is a valid external URL: ${links[i]}`,
-					);
+					// Use Puppeteer to validate link address works
+					const page = await browser.newPage();
+
+					// Set a user-agent to mimic a real browser
+					await page.setUserAgent('Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36');
+
+					try {
+						let response = null;
+						let lastRedirectStatus = null;
+						let redirectChain = [];
+
+						// Capture redirects and final response
+						page.on('request', (request) => {
+							if (request.isNavigationRequest() && request.redirectChain().length) {
+								redirectChain = request.redirectChain().map((req) => req.url());
+							}
+						});
+
+						// Capture the response
+						page.on('response', (res) => {
+							const chain = res.request().redirectChain();
+							if (chain.length > 0) {
+								redirectChain = chain.map((req) => req.url());
+								lastRedirectStatus = res.status(); // Status of the last redirect
+							}
+						});
+
+						// Try loading the URL
+						response = await page.goto(links[i], { waitUntil: 'networkidle2' }).catch(() => {
+							// Ignore rendering errors (likely binary files like PDFs)
+						});
+
+						if (response) {
+							let status = response.status();
+							const contentType = response.headers()['content-type'];
+
+							// If it's a PDF switch to direct fetching
+							if (contentType && contentType.includes('application/')) {
+								status = await page.evaluate(async (url) => {
+									const res = await fetch(url, { method: 'HEAD' });
+									return res.status;
+								}, links[i]);
+							}
+							if ((status < 200 || status > 299) && status !== 304)
+								throw `Unexpected Status Returned: ${status}`;
+						}
+					} catch (error) {
+						throw error;
+					}
+
+					// Close the headless browser tab
+					page.close();
+
 				} catch (e) {
 					let error_message;
 					if (e instanceof AggregateError) {
@@ -832,6 +871,7 @@ const e = require("express");
 		gen_redirects,
 		draft_links,
 		is_private,
+		browser,
 	) => {
 		console.log("Performing Validation and Building SEO Link List...");
 		redirects = gen_redirects;
@@ -1002,7 +1042,7 @@ const e = require("express");
 				if (links.href.length === 0) {
 					messages[file.relativePath].push("No links found in file");
 				} else {
-					await checkLinks(source_path, file, links.href, hdocbook_config, hdocbook_project);
+					await checkLinks(source_path, file, links.href, hdocbook_config, hdocbook_project, browser);
 				}
 				if (links.img.length === 0) {
 					messages[file.relativePath].push("No images found in file");

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "hdoc-tools",
-	"version": "0.34.2",
+	"version": "0.35.0",
 	"description": "Hornbill HDocBook Development Support Tool",
 	"main": "hdoc.js",
 	"bin": {