hdoc-tools 0.51.0 → 0.52.0

package/hdoc-build.js

@@ -11,10 +11,9 @@
 	const hdoc_build_onyx = require(path.join(__dirname, "hdoc-build-onyx.js"));
 	const hdoc_build_pdf = require(path.join(__dirname, "hdoc-build-pdf.js"));
 	const hdoc_index = require(path.join(__dirname, "hdoc-db.js"));
+	const hdoc_mermaid = require(path.join(__dirname, "hdoc-mermaid.js"));
 	const archiver = require("archiver");
 
-	const { execSync } = require("child_process");
-
 	const h_tags_to_search = ["h1", "h2", "h3"];
 	const image_extensions = ["png", "svg", "jpg"];
 
@@ -44,14 +43,6 @@
 		"mermaid-theme.yaml",
 	);
 
-	let mermaid_puppeteer_config_path = path.resolve(
-		templates_path,
-		"mermaid-puppeteer-config.json",
-	);
-	if (process.platform === "win32") {
-		mermaid_puppeteer_config_path = `"${mermaid_puppeteer_config_path}"`
-	}
-
 	const pdf_template_path = path.join(__dirname, "templates", "pdf");
 	const ui_css_path = path.join(__dirname, "ui", "css");
 	const pdf_template_file_path = path.join(pdf_template_path, "template.html");
@@ -70,6 +61,10 @@
 	const redirects = {};
 	const static_html_files = [];
 	const mermaid_failures = [];
+	// Mermaid diagrams collected during (synchronous) markdown rendering, then
+	// rendered to SVG in a batch afterwards (puppeteer is async; the markdown-it
+	// highlight callback is not). Keyed by output path to dedupe identical diagrams.
+	const mermaid_queue = new Map();
 
 	let bc = {}; // Breadcrumbs map
 	let book_read_time = 0;
@@ -117,38 +112,30 @@
 		typographer: true,
 		highlight: function (str, lang) {
 			if (lang === "mermaid" && process.env.GITHUB_ACTIONS !== 'true') {
-				try {
-					const tmpInput = hdoc.tmp_file_sync({ postfix: ".mmd" });
-					const outputFileName = `mermaid-${crypto.createHash("sha256").update(str).digest("hex").slice(0, 16)}.svg`;
-					const outputPath = path.join(mermaid_images_path, outputFileName);
-					const outputLink = `/_books/${doc_id}/mermaid-images/${outputFileName}`;
-
-					if (!str.startsWith('---')) {
-						str = '---\n' + fs.readFileSync(mermaid_theme_path, {encoding: 'utf-8'}) + `\n---\n${str}`;
-					}
-
-					fs.writeFileSync(tmpInput.name, str);
-					let cmd = `${__dirname}/node_modules/.bin/mmdc`;
-
-					if (process.platform === "win32") {
-						cmd = `"${cmd}.cmd"`;
-					}
-
-					cmd = `${cmd} -i "${tmpInput.name}" -o "${outputPath}" --backgroundColor transparent --puppeteerConfigFile ${mermaid_puppeteer_config_path}`;
-					console.log(`Generating Mermaid SVG found in ${currentMdFilePath.relativePath} - ${outputPath}`);
-					execSync(cmd);
-
-					if (!fs.existsSync(outputPath)) {
-						throw new Error("mmdc did not generate output");
-					}
-
-					tmpInput.removeCallback();
+				// markdown-it's highlight callback is synchronous, but Mermaid
+				// rendering is async (puppeteer). So here we only *queue* the
+				// diagram and emit the <img> tag pointing at its eventual SVG;
+				// the actual rendering happens in a batch after all markdown has
+				// been processed (see render_mermaid_queue, before validation/zip).
+				const outputFileName = `mermaid-${crypto.createHash("sha256").update(str).digest("hex").slice(0, 16)}.svg`;
+				const outputPath = path.join(mermaid_images_path, outputFileName);
+				const outputLink = `/_books/${doc_id}/mermaid-images/${outputFileName}`;
+
+				let definition = str;
+				if (!definition.startsWith('---')) {
+					definition = '---\n' + fs.readFileSync(mermaid_theme_path, {encoding: 'utf-8'}) + `\n---\n${definition}`;
+				}
 
-					return `<img class="mermaid-diagram" src="${outputLink}" alt="Mermaid Diagram">`;
-				} catch (err) {
-					mermaid_failures.push({path: currentMdFilePath.relativePath, error: err.message});
-					return ``;
+				// Dedupe identical diagrams (same hash -> same output file).
+				if (!mermaid_queue.has(outputPath)) {
+					mermaid_queue.set(outputPath, {
+						definition,
+						outputPath,
+						relativePath: currentMdFilePath.relativePath,
+					});
 				}
+
+				return `<img class="mermaid-diagram" src="${outputLink}" alt="Mermaid Diagram">`;
 			}
 		}
 	});
@@ -197,6 +184,35 @@
 	// generates a PDF, and writes the final HTML to disk ready for indexing.
 	// Markdown files are rendered to HTML first; static HTML files are processed
 	// in place. Both paths share the same post-processing pipeline.
+	// Render every queued-but-not-yet-rendered Mermaid diagram to its SVG file,
+	// reusing the already-open browser. Each diagram's render is memoized on the
+	// queue entry (item.promise), so concurrent transform_file calls and the final
+	// safety flush never render the same diagram twice. Failures are collected in
+	// mermaid_failures rather than thrown, matching the previous mmdc behavior.
+	const flush_mermaid_queue = async () => {
+		await Promise.all(
+			[...mermaid_queue.values()].map((item) => {
+				if (!item.promise) {
+					item.promise = (async () => {
+						try {
+							console.log(`Generating Mermaid SVG found in ${item.relativePath} - ${item.outputPath}`);
+							const svg = await hdoc_mermaid.render_svg(browser, item.definition, {
+								backgroundColor: "transparent",
+							});
+							if (!svg || svg.indexOf("<svg") === -1) {
+								throw new Error("Mermaid produced no SVG output");
+							}
+							fs.writeFileSync(item.outputPath, svg);
+						} catch (err) {
+							mermaid_failures.push({ path: item.relativePath, error: err.message });
+						}
+					})();
+				}
+				return item.promise;
+			}),
+		);
+	};
+
 	const transform_file = async (file_path) => {
 		const is_markdown = path.extname(file_path.path) === '.md';
 		if (is_markdown) conversion_attempted++;
@@ -254,6 +270,11 @@
 			// Render markdown into HTML
 			html_txt = md.render(md_txt);
 
+			// md.render() synchronously queued any Mermaid diagrams in this file.
+			// Render them to SVG now (memoized/deduped) so the files exist before
+			// PDF generation below reads them, and before validation/zipping.
+			await flush_mermaid_queue();
+
 			// Single pass: wrap h2/h3 divs + extract heading, paragraph, read-time.
 			// Replaces separate wrapHContent + getFirstHTMLHeading + get_html_read_time calls.
 			const extracted = hdoc.wrapAndExtract(html_txt, h_tags_to_search);
@@ -1279,6 +1300,10 @@
 			);
 		}
 
+		// Safety net: render any Mermaid diagrams not already flushed during
+		// transform_file (no-op if all are done — rendering is memoized per diagram).
+		await flush_mermaid_queue();
+
 		// Output to console
 		console.log(`\n                MD files found: ${conversion_attempted}`);
 		console.log(`Successfully converted to HTML: ${conversion_success}`);
@@ -1418,7 +1443,7 @@
 				const zip_path = path.join(work_path, `${doc_id}.zip`);
 
 				const output = fs.createWriteStream(zip_path);
-				const archive = archiver("zip");
+				const archive = new archiver.ZipArchive();
 				archive.on("error", (err) => {
 					throw err;
 				});

package/hdoc-help.js

@@ -15,7 +15,10 @@ Commands
      - Use the '--no-links' argument to skip link output to CLI during validation.
 
  - createDocs
-    Creates folder structure and markdown documents as defined in the HDocBook navigation item links 
+    Creates folder structure and markdown documents as defined in the HDocBook navigation item links
+
+ - edit
+    Starts the local structure/content editor on port 3000 (bound to 127.0.0.1), serving the editor UI alongside a live preview of the content. Supports a -port N to use a different port.
 
  - help
     Outputs available arguments and switches

package/hdoc-mermaid.js

@@ -0,0 +1,204 @@
+// In-process Mermaid -> SVG renderer.
+//
+// Replaces the previous `@mermaid-js/mermaid-cli` (mmdc) integration. Instead of
+// shelling out to the CLI (which spawned a fresh Chromium per diagram and pinned
+// puppeteer to <=24 via its peerDependency), this renders Mermaid using the
+// `mermaid` library directly inside a puppeteer browser supplied by the caller —
+// the very same browser hdoc-build already launches for PDF/link work.
+//
+// The render path (Interceptor + page evaluation) is ported from mermaid-cli
+// (MIT licensed): https://github.com/mermaid-js/mermaid-cli
+// Trimmed to the SVG-only case we need, plus ELK layout support (required by
+// books that use `layout: elk`). zenuml / tidy-tree / icon-packs / png / pdf are
+// intentionally dropped — no hdocbook content uses them.
+
+const { readFile, realpath } = require("node:fs/promises");
+const path = require("node:path");
+const url = require("node:url");
+
+// ESM bundle locations inside the installed packages. Both packages expose
+// "./*": "./*" in their exports map, so the dist files resolve directly.
+const mermaidESMPath = require.resolve("mermaid/dist/mermaid.esm.mjs");
+const elkESMPath = require.resolve(
+	"@mermaid-js/layout-elk/dist/mermaid-layout-elk.esm.mjs",
+);
+
+/**
+ * Guesses the MIME-type of a file based on its extension. Only the handful of
+ * types the Mermaid ESM bundles actually request are supported.
+ */
+function getContentTypeFromFileExtension(filePath) {
+	const ext = path.extname(filePath).toLowerCase();
+	switch (ext) {
+		case ".css":
+			// Force UTF-8 so puppeteer doesn't mis-parse as Latin-1.
+			return "text/css;charset=UTF-8";
+		case ".js":
+		case ".mjs":
+			return "application/javascript";
+		case ".woff2":
+			return "font/woff2";
+		default:
+			throw new Error(`Unsupported file extension for intercept: ${ext}`);
+	}
+}
+
+/**
+ * Puppeteer can't `import` ESM modules from `file://` URLs. Rather than stand up
+ * a real HTTP server (ports/firewalls), we intercept requests to a dummy
+ * `https://hdoc-mermaid-intercept.invalid` origin and serve the corresponding
+ * local file. Ported verbatim from mermaid-cli's puppeteerIntercept.js.
+ */
+class Interceptor {
+	#INTERCEPT_ORIGIN = "https://hdoc-mermaid-intercept.invalid";
+
+	/** @type {Set<string>} resolved (realpath) dirs allowed to be served. */
+	#allowedDirs = new Set();
+
+	async fileUrlToInterceptUrl(fileUrl, { allowParentDirectoryLevel = 1 } = {}) {
+		fileUrl = new URL(fileUrl);
+		if (fileUrl.protocol !== "file:") {
+			throw new Error(`Invalid file URL: ${fileUrl}`);
+		}
+		let parentDirectory = await realpath(url.fileURLToPath(fileUrl));
+		while (allowParentDirectoryLevel-- >= 0) {
+			parentDirectory = path.dirname(parentDirectory);
+		}
+		this.#allowedDirs.add(parentDirectory);
+		return `${this.#INTERCEPT_ORIGIN}${fileUrl.pathname}`;
+	}
+
+	async interceptUrlToFileUrl(interceptUrl) {
+		interceptUrl = new URL(interceptUrl);
+		if (interceptUrl.origin !== this.#INTERCEPT_ORIGIN) {
+			throw new Error(`Invalid intercept URL: ${interceptUrl}`);
+		}
+		const fileUrl = new URL(
+			interceptUrl.href.slice(this.#INTERCEPT_ORIGIN.length),
+			"file://",
+		);
+		const filePath = await realpath(url.fileURLToPath(fileUrl));
+		if (
+			![...this.#allowedDirs].some((dir) =>
+				path.relative(filePath, dir).startsWith(".."),
+			)
+		) {
+			throw new Error(
+				`Intercept URL is not in an allowed directory: ${interceptUrl}`,
+			);
+		}
+		return fileUrl;
+	}
+
+	async #interceptRequestHandler(request) {
+		try {
+			if (request.url().startsWith(this.#INTERCEPT_ORIGIN)) {
+				const fileUrl = await this.interceptUrlToFileUrl(request.url());
+				return request.respond({
+					status: 200,
+					headers: { "Access-Control-Allow-Origin": "*" },
+					contentType: getContentTypeFromFileExtension(
+						url.fileURLToPath(fileUrl),
+					),
+					body: await readFile(fileUrl),
+				});
+			}
+		} catch (error) {
+			console.error(
+				`Error handling intercept request for ${request.url()}:`,
+				error,
+			);
+			request.abort();
+			return;
+		}
+		request.continue();
+	}
+
+	get interceptRequestHandler() {
+		return this.#interceptRequestHandler.bind(this);
+	}
+}
+
+/**
+ * Render a single Mermaid definition to an SVG string.
+ *
+ * @param {import('puppeteer').Browser} browser - An open puppeteer browser.
+ * @param {string} definition - Mermaid source (may include `--- config: ---` frontmatter).
+ * @param {{ backgroundColor?: string }} [opts]
+ * @returns {Promise<string>} the serialized `<svg>`.
+ */
+async function render_svg(browser, definition, { backgroundColor = "transparent" } = {}) {
+	const page = await browser.newPage();
+	// Surface in-page warnings/errors (e.g. mermaid parse errors) to the build log.
+	page.on("console", (msg) => {
+		if (msg.type() === "error" || msg.type() === "warning") {
+			console.warn(msg.text());
+		}
+	});
+	try {
+		await page.setContent(
+			"<!doctype html><html><body><div id=\"container\"></div></body></html>",
+		);
+		await page.$eval(
+			"body",
+			(body, bg) => {
+				body.style.background = bg;
+			},
+			backgroundColor,
+		);
+
+		const interceptor = new Interceptor();
+		const mermaidUrl = await interceptor.fileUrlToInterceptUrl(
+			url.pathToFileURL(mermaidESMPath),
+		);
+		const elkUrl = await interceptor.fileUrlToInterceptUrl(
+			url.pathToFileURL(elkESMPath),
+		);
+
+		page.on("request", interceptor.interceptRequestHandler);
+		await page.setRequestInterception(true);
+
+		await page.$eval(
+			"#container",
+			async (container, { definition, mermaidUrl, elkUrl, backgroundColor }) => {
+				const { default: mermaid } = await import(mermaidUrl);
+				const { default: elkLayouts } = await import(elkUrl);
+
+				await Promise.all(
+					Array.from(document.fonts, (font) => font.load()),
+				);
+
+				mermaid.registerLayoutLoaders(elkLayouts);
+				mermaid.initialize({ startOnLoad: false });
+
+				// Throws on invalid diagrams — propagates out of $eval as a rejection.
+				const { svg: svgText } = await mermaid.render(
+					"hdoc-svg",
+					definition,
+					container,
+				);
+				container.innerHTML = svgText;
+
+				const svg = container.getElementsByTagName?.("svg")?.[0];
+				if (svg?.style) {
+					svg.style.backgroundColor = backgroundColor;
+				}
+			},
+			{ definition, mermaidUrl, elkUrl, backgroundColor },
+		);
+
+		// Serialize via XMLSerializer so HTML <foreignObject> content (e.g. <br>)
+		// becomes valid XML in the saved .svg file.
+		const svgXML = await page.$eval("svg", (svg) => {
+			// eslint-disable-next-line no-undef
+			const xmlSerializer = new XMLSerializer();
+			return xmlSerializer.serializeToString(svg);
+		});
+
+		return svgXML;
+	} finally {
+		await page.close();
+	}
+}
+
+module.exports = { render_svg, Interceptor };

package/hdoc-module.js

@@ -9,10 +9,48 @@
 
 	let retried = false;
 
+	const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
+
+	// GitHub enforces a "secondary" (burst) rate limit on top of the hourly one:
+	// too many requests in a short window are rejected with 403/429 even when the
+	// hourly budget is intact. The contributor pull fires one call per file, so a
+	// fast run trips it. Historically `hdoc validate` got away with this only
+	// because verbose console output slowed the loop down between calls; the
+	// --quiet mode removed that incidental spacing and started hitting 403s.
+	// Throttle GitHub API calls to a fixed minimum interval so behaviour no longer
+	// depends on how chatty the console is.
+	let _last_github_call_at = 0;
+	const GITHUB_HOST = "api.github.com";
+	const GITHUB_MIN_INTERVAL_MS = 800;
+	// Never block the whole build waiting on a single rate-limited call.
+	const GITHUB_MAX_BACKOFF_MS = 60000;
+
+	// For a rate-limited response, work out how long to wait before retrying.
+	// Returns null when the response is NOT a retryable rate limit (e.g. a genuine
+	// 403 auth/forbidden), in which case the caller should give up immediately.
+	const rate_limit_wait_ms = (response) => {
+		if (response.status !== 403 && response.status !== 429) return null;
+		const retry_after = response.headers.get("retry-after");
+		if (retry_after !== null) {
+			const secs = Number(retry_after);
+			if (!Number.isNaN(secs)) return Math.max(0, secs * 1000);
+		}
+		const remaining = response.headers.get("x-ratelimit-remaining");
+		const reset = response.headers.get("x-ratelimit-reset");
+		if (remaining === "0" && reset !== null) {
+			const reset_ms = Number(reset) * 1000 - Date.now();
+			if (!Number.isNaN(reset_ms)) return Math.max(0, reset_ms);
+		}
+		// 403/429 with no rate-limit signal => auth/forbidden, not worth retrying.
+		return null;
+	};
+
 	// Wraps the built-in fetch() with automatic retry for transient errors.
 	// Retries up to maxRetries times when the server returns an HTTP error status
-	// >= 400, except for 401 (Unauthorized) and 403 (Forbidden) which are auth
-	// failures that won't be resolved by retrying. Network errors (where fetch
+	// >= 400, except for 401 (Unauthorized) and non-rate-limit 403 (Forbidden)
+	// which are auth failures that won't be resolved by retrying. GitHub
+	// rate-limit responses (403/429 carrying Retry-After or X-RateLimit-Reset) ARE
+	// retried after honouring the indicated wait. Network errors (where fetch
 	// itself throws) are also retried. Sets the module-level `retried` flag so
 	// callers can detect and log a success-after-retry message.
 	//
@@ -21,8 +59,18 @@
 	// does not leave an already-aborted signal in place for the retries.
 	const fetchWithRetry = async (url, options = {}, maxRetries = 5) => {
 		const { timeoutMs, ...fetchOptions } = options;
+		const is_github = typeof url === "string" && url.includes(GITHUB_HOST);
 		let retryCount = 0;
 		while (true) {
+			// Space out GitHub API calls so a fast (quiet) run does not burst past
+			// the secondary rate limit. No effect on non-GitHub requests.
+			if (is_github) {
+				const since = Date.now() - _last_github_call_at;
+				if (since < GITHUB_MIN_INTERVAL_MS) {
+					await sleep(GITHUB_MIN_INTERVAL_MS - since);
+				}
+				_last_github_call_at = Date.now();
+			}
 			// Create a fresh signal for each attempt; reusing an already-aborted
 			// signal would cause every subsequent retry to abort immediately.
 			const attemptOptions = timeoutMs
@@ -38,10 +86,19 @@
 				retried = true;
 				continue;
 			}
-			// Auth failures and successes are not retried
-			if (response.ok || response.status === 401 || response.status === 403) {
+			if (response.ok || response.status === 401) {
 				return response;
 			}
+			// 403/429: retry only when it is a rate limit we can wait out.
+			if (response.status === 403 || response.status === 429) {
+				const wait_ms = rate_limit_wait_ms(response);
+				if (wait_ms === null) return response; // auth/forbidden — give up
+				retryCount++;
+				if (retryCount > maxRetries) return response;
+				retried = true;
+				await sleep(Math.min(wait_ms, GITHUB_MAX_BACKOFF_MS));
+				continue;
+			}
 			retryCount++;
 			if (retryCount > maxRetries) return response;
 			retried = true;