hdoc-tools 0.47.3 → 0.47.4

package/hdoc-module.js

@@ -48,6 +48,8 @@
 		}
 	};
 
+	exports.fetchWithRetry = fetchWithRetry;
+
 	exports.content_type_for_ext = (ext) => {
 		switch (ext) {
 			case ".z":

package/hdoc-validate.js

@@ -465,14 +465,35 @@ const { error } = require("node:console");
 		return returnPaths;
 	}
 
-	const _fetch_headers = { 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36' };
+	// Headers that mimic a real Chrome browser request — sites doing bot detection
+	// check far more than just User-Agent (Accept, Sec-Fetch-*, client hints, etc.).
+	const _fetch_headers = {
+		'User-Agent':      'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36',
+		'Accept':          'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8',
+		'Accept-Language': 'en-US,en;q=0.9',
+		'Accept-Encoding': 'gzip, deflate, br',
+		'Cache-Control':   'no-cache',
+		'Pragma':          'no-cache',
+		'Sec-Fetch-Dest':  'document',
+		'Sec-Fetch-Mode':  'navigate',
+		'Sec-Fetch-Site':  'none',
+		'Sec-Fetch-User':  '?1',
+		'Sec-Ch-Ua':       '"Google Chrome";v="131", "Chromium";v="131", "Not_A Brand";v="24"',
+		'Sec-Ch-Ua-Mobile':   '?0',
+		'Sec-Ch-Ua-Platform': '"Windows"',
+		'Upgrade-Insecure-Requests': '1',
+	};
 
-	// Checks a single external URL by sending a HEAD request (falling back to GET
-	// if the server returns 405 Method Not Allowed). Returns the HTTP status code.
+	// Checks a single external URL by sending a HEAD request, falling back to GET
+	// if the server returns 405 (Method Not Allowed) or 404 (some servers, e.g.
+	// marketplace.visualstudio.com, return 404 for HEAD even when the page exists).
+	// Retries up to 5 times on transient errors (5xx, 429, network failures).
+	// Returns the HTTP status code.
 	const fetchExternalLinkStatus = async (url) => {
-		const resp = await fetch(url, { method: 'HEAD', headers: _fetch_headers, signal: AbortSignal.timeout(10000), redirect: 'follow' });
-		if (resp.status === 405) {
-			const getResp = await fetch(url, { method: 'GET', headers: _fetch_headers, signal: AbortSignal.timeout(10000), redirect: 'follow' });
+		const opts = { method: 'HEAD', headers: _fetch_headers, timeoutMs: 10000, redirect: 'follow' };
+		const resp = await hdoc.fetchWithRetry(url, opts);
+		if (resp.status === 404 || resp.status === 405) {
+			const getResp = await hdoc.fetchWithRetry(url, { ...opts, method: 'GET' });
 			return getResp.status;
 		}
 		return resp.status;

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "hdoc-tools",
-	"version": "0.47.3",
+	"version": "0.47.4",
 	"description": "Hornbill HDocBook Development Support Tool",
 	"main": "hdoc.js",
 	"bin": {