hdoc-tools 0.64.0 → 0.65.0

package/hdoc-edit.js

@@ -247,7 +247,9 @@
 		app.get("/api/events", (req, res) => {
 			res.writeHead(200, {
 				"Content-Type": "text/event-stream",
-				"Cache-Control": "no-cache",
+				// no-transform: the compression middleware otherwise buffers
+				// the stream and events never reach the client
+				"Cache-Control": "no-cache, no-transform",
 				Connection: "keep-alive",
 			});
 			res.write(": connected\n\n");

package/hdoc-serve-validate.js

@@ -0,0 +1,516 @@
+// Lightweight in-editor validation for the `hdoc serve` Edit Mode (PoC).
+//
+// NOT a replacement for `hdoc validate`: that runs the full build pipeline
+// (hdoc-build.js) over the whole book and stays the source of truth — the
+// Edit Mode "Validate book" action spawns it as a child process. What lives
+// here are the FAST, targeted subsets suited to an editing loop:
+//
+//   validate_nav(...)  — the navigation-tree checks (path validity, target
+//                        exists, on-disk case match, British spellings in
+//                        labels and path segments, nesting depth), per node.
+//   validate_page(...) — single-article checks over the page's markdown and
+//                        its RENDERED html (same pipeline as published):
+//                        spellcheck, filename rule, H1 count, image alt +
+//                        existence, link rules incl. same-page anchors and
+//                        optional external HTTP checks.
+//
+// Check semantics mirror hdoc-validate.js; cross-book links and inter-book
+// anchors need the GitHub-backed machinery, so they are reported as skipped
+// here and left to the full validate.
+
+const fs = require("node:fs");
+const path = require("node:path");
+const cheerio = require("cheerio");
+const hdoc = require(path.join(__dirname, "hdoc-module.js"));
+const { dialect_findings } = require(path.join(__dirname, "hdoc-spell.js"));
+
+const DOC_FILENAME_RE = /^[a-z]+[-a-z0-9]+$/;
+
+// --- shared helpers ---------------------------------------------------------
+
+const line_col_of = (text, offset) => {
+	let line = 1;
+	let last_nl = -1;
+	for (let i = 0; i < offset && i < text.length; i++) {
+		if (text[i] === "\n") {
+			line++;
+			last_nl = i;
+		}
+	}
+	return { line, col: offset - last_nl };
+};
+
+// Case-exact check for a source-relative file path. Returns null when every
+// segment matches the on-disk casing (or does not exist at all — existence is
+// its own check), else the offending segment pair.
+const case_mismatch = (root, rel) => {
+	let cur = root;
+	for (const seg of rel.split("/")) {
+		let entries;
+		try {
+			entries = fs.readdirSync(cur);
+		} catch {
+			return null;
+		}
+		if (entries.includes(seg)) {
+			cur = path.join(cur, seg);
+			continue;
+		}
+		const ci = entries.find((e) => e.toLowerCase() === seg.toLowerCase());
+		return ci ? { expected: ci, got: seg } : null;
+	}
+	return null;
+};
+
+// Resolve a book link (leading-slash tolerant, no extension) to an existing
+// source file. Returns the source-relative path or null.
+const resolve_link_target = (source_path, link) => {
+	const clean = String(link).replace(/^\/+/, "").split("#")[0];
+	if (!clean) return null;
+	for (const rel of [
+		`${clean}.md`,
+		`${clean}/index.md`,
+		`${clean}.html`,
+		`${clean}.htm`,
+		`${clean}/index.html`,
+		`${clean}/index.htm`,
+	]) {
+		if (fs.existsSync(path.join(source_path, rel))) return rel;
+	}
+	return null;
+};
+
+const redirect_for = (project, link) => {
+	const list =
+		project && Array.isArray(project.redirects) ? project.redirects : [];
+	const norm = link.startsWith("/") ? link : `/${link}`;
+	return list.find((r) => {
+		const u = String(r.url || "");
+		return (u.startsWith("/") ? u : `/${u}`) === norm;
+	});
+};
+
+const project_dictionary = (project) => {
+	const v = project && project.validation;
+	return v && Array.isArray(v.spellcheckDictionary)
+		? v.spellcheckDictionary
+		: [];
+};
+
+// Per-document spellcheck exclusions ({document_path, words}) for a logical
+// page path (no extension).
+const doc_spell_excludes = (project, logical) => {
+	const v = project && project.validation;
+	const list = v && Array.isArray(v.exclude_spellcheck) ? v.exclude_spellcheck : [];
+	const hit = list.find(
+		(e) => String(e.document_path).replace(/^\/+/, "") === logical,
+	);
+	return hit && Array.isArray(hit.words) ? hit.words : [];
+};
+
+// --- navigation validation ---------------------------------------------------
+
+// items: hdocbook navigation items (with ids stamped by TocModel).
+// Returns [{ id, problems: [string] }] for nodes with findings only.
+exports.validate_nav = (items, source_path, docId, project) => {
+	const dict = project_dictionary(project);
+	const out = [];
+
+	const spell = (text) =>
+		dialect_findings(String(text || ""), dict).map(
+			(f) => `British spelling: "${f.word}" should be "${f.suggestion}"`,
+		);
+
+	const walk = (nodes, depth) => {
+		for (const n of nodes) {
+			const problems = [];
+
+			if (depth > 5) {
+				problems.push("Navigation is nested deeper than 5 levels");
+			}
+
+			for (const p of spell(n.text)) problems.push(`Label: ${p}`);
+
+			const link = typeof n.link === "string" ? n.link : null;
+			if (link) {
+				const clean = link.replace(/^\/+/, "");
+				if (/[^a-zA-Z0-9\-/]/.test(clean)) {
+					problems.push(`Link contains invalid characters: ${link}`);
+				}
+				if (!(clean === docId || clean.startsWith(`${docId}/`))) {
+					problems.push(
+						`Link points outside this book (${docId}): ${link}`,
+					);
+				} else {
+					const rel = resolve_link_target(source_path, clean);
+					if (!rel) {
+						if (!redirect_for(project, clean)) {
+							problems.push(`Target file does not exist: ${link}`);
+						}
+					} else {
+						const cm = case_mismatch(source_path, rel);
+						if (cm) {
+							problems.push(
+								`Path case does not match disk: "${cm.got}" is "${cm.expected}" on disk`,
+							);
+						}
+					}
+					// British spellings inside the path segments
+					const seg_text = clean.split(/[-/]/).join(" ");
+					for (const p of spell(seg_text)) problems.push(`Path: ${p}`);
+				}
+			}
+
+			if (problems.length) out.push({ id: n.id, problems });
+			if (Array.isArray(n.items)) walk(n.items, depth + 1);
+		}
+	};
+	walk(items, 1);
+	return out;
+};
+
+// --- inline (as-you-type) lint ------------------------------------------------
+
+// Character ranges of fenced code blocks and inline code spans, so link
+// findings never fire inside code (the build never renders those as links).
+const code_ranges = (md) => {
+	const ranges = [];
+	const fence_re = /^(```|~~~)[^\n]*\n[\s\S]*?(?:^\1\s*$|(?![\s\S]))/gm;
+	let m;
+	while ((m = fence_re.exec(md)) !== null) {
+		ranges.push([m.index, m.index + m[0].length]);
+	}
+	const tick_re = /`[^`\n]+`/g;
+	while ((m = tick_re.exec(md)) !== null) {
+		ranges.push([m.index, m.index + m[0].length]);
+	}
+	return ranges;
+};
+
+const in_ranges = (ranges, pos) =>
+	ranges.some(([a, b]) => pos >= a && pos < b);
+
+// Fast, offset-based findings over an UNSAVED markdown buffer, for editor
+// underlines: British spellings (with the US suggestion as a one-click fix)
+// and syntactic/local link problems. No HTTP — external URLs are left to
+// the on-demand page/book validation.
+// args: { md, source_path, docId, project, logical }
+// -> [{ from, to, severity: 'error'|'warning', message, fix? }]
+exports.lint_markdown = (args) => {
+	const { md, source_path, docId, project, logical } = args;
+	const findings = [];
+
+	// British spellings (same exclusions as validate)
+	const ignore = [
+		...project_dictionary(project),
+		...doc_spell_excludes(project, logical || ""),
+	];
+	for (const f of dialect_findings(md, ignore)) {
+		findings.push({
+			from: f.from,
+			to: f.to,
+			severity: "error",
+			message: `British spelling: ${f.word} should be ${f.suggestion}`,
+			fix: f.suggestion,
+			word: f.word,
+		});
+	}
+
+	// markdown links/images: [text](target) / ![alt](src)
+	const code = code_ranges(md);
+	const link_re = /(!?)\[[^\]]*\]\(\s*<?([^)\s>]+)/g;
+	let m;
+	while ((m = link_re.exec(md)) !== null) {
+		if (in_ranges(code, m.index)) continue;
+		const is_img = m[1] === "!";
+		const target = m[2];
+		const from = m.index + m[0].length - target.length;
+		const to = from + target.length;
+		const err = (message) =>
+			findings.push({ from, to, severity: "error", message });
+
+		if (hdoc.valid_url(target)) {
+			const lower = target.toLowerCase();
+			if (
+				!is_img &&
+				(lower.includes("docs.hornbill.com") ||
+					lower.includes("docs-internal.hornbill.com"))
+			) {
+				err(`Hornbill Docs links should not be fully-qualified`);
+			}
+			continue; // external — checked by page/book validation only
+		}
+		if (target.startsWith("#")) continue; // anchor — needs a render to verify
+		if (target.startsWith("mailto:")) continue;
+
+		if (!target.startsWith("/")) {
+			err(
+				`${is_img ? "Image" : "Link"} paths should be root-relative (start with /)`,
+			);
+			continue;
+		}
+
+		const no_hash = target.split("#")[0];
+		const segments = no_hash.replace(/^\/+/, "").split("/");
+		const is_books = segments[0] === "_books";
+		const root = is_books ? segments[1] : segments[0];
+		if (!root) continue; // "/" = docs home
+
+		if (is_img) {
+			const rel = no_hash.replace(/^\/+/, "").replace(/^_books\//, "");
+			if (!fs.existsSync(path.join(source_path, rel))) {
+				err(`Image file does not exist`);
+			}
+			continue;
+		}
+
+		const ext = path.extname(no_hash).toLowerCase();
+		if (is_books && ext === "") {
+			err(`Page links should not include _books in the path`);
+			continue;
+		}
+		if (root !== docId) continue; // cross-book — full validate territory
+		if ([".md", ".html", ".htm"].includes(ext)) {
+			err(`Links should not include a file extension`);
+			continue;
+		}
+		if (ext !== "") continue; // linked resource
+
+		const clean = no_hash.replace(/^\/+/, "").replace(/^_books\//, "");
+		const resolved = resolve_link_target(source_path, clean);
+		if (!resolved) {
+			if (!redirect_for(project, clean)) err(`Link target does not exist`);
+			continue;
+		}
+		const cm = case_mismatch(source_path, resolved);
+		if (cm) {
+			err(`Path case does not match disk ("${cm.got}" is "${cm.expected}")`);
+		}
+	}
+
+	findings.sort((a, b) => a.from - b.from);
+	return findings;
+};
+
+// --- single-article validation -----------------------------------------------
+
+// External URL check: HEAD, falling back to GET on 404/405 (same approach as
+// hdoc-validate.js), shorter retry budget — this is an interactive loop.
+const check_external = async (url) => {
+	const opts = {
+		method: "HEAD",
+		headers: { "user-agent": "hdoc-serve-validate" },
+		timeoutMs: 8000,
+		redirect: "follow",
+	};
+	let resp = await hdoc.fetchWithRetry(url, opts, 1);
+	if (resp.status === 404 || resp.status === 405) {
+		resp = await hdoc.fetchWithRetry(url, { ...opts, method: "GET" }, 1);
+	}
+	return resp.status;
+};
+
+// args: { source_path, docId, rel (source-relative md path), logical,
+//         md, html (rendered page fragment), project, external (bool) }
+// Returns { errors: [], warnings: [], skipped: [] } — strings, prefixed
+// "line:col - " where known.
+exports.validate_page = async (args) => {
+	const {
+		source_path,
+		docId,
+		rel,
+		logical,
+		md,
+		html,
+		project,
+		external,
+	} = args;
+	const errors = [];
+	const warnings = [];
+	const skipped = [];
+
+	// 1) filename convention (mirrors hdoc-build.js; _-prefixed and *_ext.md
+	// files never reach here — resolve_page_md refuses them)
+	const stem = path.basename(rel).replace(/\.md$/i, "");
+	if (!DOC_FILENAME_RE.test(stem) && stem !== "index") {
+		errors.push(
+			`Filename does not meet naming standards (^[a-z]+[-a-z0-9]+$): ${path.basename(rel)}`,
+		);
+	}
+
+	// 2) British spellings over the markdown source
+	const ignore = [
+		...project_dictionary(project),
+		...doc_spell_excludes(project, logical),
+	];
+	for (const f of dialect_findings(md, ignore)) {
+		const { line, col } = line_col_of(md, f.from);
+		errors.push(
+			`${line}:${col} - British spelling: ${f.word} should be ${f.suggestion}`,
+		);
+	}
+
+	// 3) rendered-output checks
+	const $ = cheerio.load(html);
+
+	// H1 rule (page fragment: the generated document header holds the one
+	// legitimate h1; a second one comes from the content)
+	const v = project && project.validation ? project.validation : {};
+	const h1_excluded =
+		Array.isArray(v.exclude_h1_count) &&
+		v.exclude_h1_count.some(
+			(p) => String(p).replace(/^\/+/, "") === logical,
+		);
+	const h1_count = $("h1").length;
+	if (h1_count > 1 && !h1_excluded) {
+		errors.push(
+			`Page contains ${h1_count} <h1> headings — only one # heading is allowed per page`,
+		);
+	}
+
+	// images
+	$("img").each((_i, el) => {
+		const src = String($(el).attr("src") || "");
+		const alt = $(el).attr("alt");
+		if (alt === undefined || String(alt).trim() === "") {
+			errors.push(`Image has a missing or empty alt attribute: ${src}`);
+		}
+		if (!src) return;
+		if (hdoc.valid_url(src)) {
+			skipped.push(`External image not checked: ${src}`);
+			return;
+		}
+		if (!src.startsWith("/")) {
+			errors.push(
+				`Root relative image links should start with a forward-slash: ${src}`,
+			);
+			return;
+		}
+		const img_rel = src.replace(/^\/+/, "").replace(/^_books\//, "");
+		if (!fs.existsSync(path.join(source_path, img_rel.split("#")[0]))) {
+			errors.push(`Image file does not exist: ${src}`);
+		}
+	});
+
+	// links
+	const anchors_ok = (hash) => {
+		const clean = hash.replace(/^\/?#/, "");
+		return $(`div#hb-doc-anchor-${clean.replace(/^hb-doc-anchor-/, "")}`).length > 0 ||
+			html.includes(`id="hb-doc-anchor-${clean}"`);
+	};
+	const externals = [];
+	const seen = new Set();
+	$("a[href]").each((_i, el) => {
+		const href = String($(el).attr("href") || "").trim();
+		if (!href || seen.has(href)) return;
+		seen.add(href);
+
+		if (href.startsWith("#") || href.startsWith("/#")) {
+			if (!anchors_ok(href)) {
+				errors.push(
+					`Target hash anchor is not present in page content: ${href}`,
+				);
+			}
+			return;
+		}
+
+		const valid = hdoc.valid_url(href);
+		if (valid) {
+			if (valid.protocol === "mailto:") return;
+			const lower = href.toLowerCase();
+			if (
+				lower.includes("docs.hornbill.com") ||
+				lower.includes("docs-internal.hornbill.com")
+			) {
+				errors.push(
+					`Hornbill Docs links should not be fully-qualified: ${href}`,
+				);
+				return;
+			}
+			if (external) externals.push(href);
+			else skipped.push(`External link not checked: ${href}`);
+			return;
+		}
+
+		if (!href.startsWith("/")) {
+			errors.push(
+				`Root relative links should start with a forward-slash: ${href}`,
+			);
+			return;
+		}
+
+		const segments = href.replace(/^\/+/, "").split("/");
+		const is_books = segments[0] === "_books";
+		const root = is_books ? segments[1] : segments[0];
+		if (!root) return; // bare "/" = docs home
+
+		if (is_books && path.extname(href) === "") {
+			errors.push(
+				`Root relative page links should not include _books in the path: ${href}`,
+			);
+			return;
+		}
+
+		if (root !== docId) {
+			skipped.push(
+				`Cross-book link not checked (run a full validate): ${href}`,
+			);
+			return;
+		}
+
+		const no_hash = href.split("#")[0];
+		const ext = path.extname(no_hash).toLowerCase();
+		if ([".md", ".html", ".htm"].includes(ext)) {
+			errors.push(
+				`Relative links should not include a file extension: ${href}`,
+			);
+			return;
+		}
+		if (ext !== "") {
+			skipped.push(`Linked resource not checked: ${href}`);
+			return;
+		}
+
+		const clean = no_hash.replace(/^\/+/, "").replace(/^_books\//, "");
+		const target = resolve_link_target(source_path, clean);
+		if (!target) {
+			if (!redirect_for(project, clean)) {
+				errors.push(`Link target does not exist: ${href}`);
+			}
+			return;
+		}
+		const cm = case_mismatch(source_path, target);
+		if (cm) {
+			errors.push(
+				`Link path case does not match disk ("${cm.got}" is "${cm.expected}"): ${href}`,
+			);
+		}
+		if (href.includes("#")) {
+			skipped.push(
+				`Anchor on another page not checked (run a full validate): ${href}`,
+			);
+		}
+	});
+
+	// external HTTP checks, concurrent
+	if (externals.length) {
+		await Promise.all(
+			externals.map(async (url) => {
+				try {
+					const status = await check_external(url);
+					if ((status < 200 || status > 299) && status !== 304) {
+						warnings.push(
+							`External link returned HTTP ${status}: ${url}`,
+						);
+					}
+				} catch (e) {
+					warnings.push(
+						`External link unreachable (${String((e && e.message) || e)}): ${url}`,
+					);
+				}
+			}),
+		);
+	}
+
+	return { errors, warnings, skipped };
+};