hdoc-tools 0.64.0 → 0.65.0

package/hdoc-serve.js

@@ -7,6 +7,11 @@
 	const hdoc = require(path.join(__dirname, "hdoc-module.js"));
 	const { create_content_handler, build_nav_inline, resolve_within_root } =
 		require(path.join(__dirname, "hdoc-content-routes.js"));
+	const { TocModel } = require(path.join(__dirname, "hdoc-toc.js"));
+	const serve_validate = require(
+		path.join(__dirname, "hdoc-serve-validate.js"),
+	);
+	const { spawn } = require("node:child_process");
 
 	let port = 3000;
 	let docId;
@@ -202,6 +207,7 @@
 				}
 				try {
 					fs.writeFileSync(resolved.abs, body.content, "utf8");
+					note_write(resolved.abs, body.content);
 				} catch (e) {
 					return res
 						.status(500)
@@ -215,6 +221,635 @@
 				});
 			});
 
+			// --- Edit suite (PoC): nav + files management -------------------
+			//
+			// Mirrors the /api/toc and /api/files surface of `hdoc edit`,
+			// backed by the same TocModel, so the viewer's Edit Mode can
+			// manage the left-hand nav (Contents) and browse/manage the
+			// on-disk pages (Files). All routes sit under /_edit and are
+			// therefore loopback-gated by the middleware above.
+			//
+			// The model is created lazily on first use and re-read from disk
+			// on every GET /_edit/toc (the tree fetch that precedes any run
+			// of edits), so external hdocbook.json edits are picked up. It is
+			// deliberately NOT reloaded before mutations: freshly assigned
+			// node ids only persist on the first structural save, and a
+			// reload in between would regenerate them and orphan the client.
+			let toc = null;
+			const get_toc = () => {
+				if (!toc) {
+					toc = new TocModel(global_source_path, docId);
+					// suppress our own hdocbook.json writes in the watcher
+					toc.on_persist = note_write;
+				}
+				return toc;
+			};
+			const toc_op = (res, fn) => {
+				try {
+					res.json(fn(get_toc()));
+				} catch (e) {
+					res.status(400).json({ error: String((e && e.message) || e) });
+				}
+			};
+
+			app.get("/_edit/toc", (req, res) =>
+				toc_op(res, (t) => {
+					t.reload();
+					return t.to_dto();
+				}),
+			);
+
+			app.post("/_edit/toc/move", (req, res) => {
+				const b = req.body || {};
+				toc_op(res, (t) => {
+					if (!b.id) throw new Error("Missing id");
+					t.move(b.id, b.parentId ?? null, b.index);
+					return t.to_dto();
+				});
+			});
+
+			app.post("/_edit/toc/rename", (req, res) => {
+				const b = req.body || {};
+				toc_op(res, (t) => {
+					if (!b.id) throw new Error("Missing id");
+					if (typeof b.text !== "string") throw new Error("Missing text");
+					t.rename(b.id, b.text);
+					return t.to_dto();
+				});
+			});
+
+			app.post("/_edit/toc/draft", (req, res) => {
+				const b = req.body || {};
+				toc_op(res, (t) => {
+					if (!b.id) throw new Error("Missing id");
+					t.set_draft(b.id, !!b.draft);
+					return t.to_dto();
+				});
+			});
+
+			// Add a nav leaf linking to a page (optionally creating the file).
+			app.post("/_edit/toc/create", (req, res) => {
+				const b = req.body || {};
+				toc_op(res, (t) => {
+					if (typeof b.link !== "string" || !b.link) {
+						throw new Error("Missing link");
+					}
+					if (b.createFile) {
+						const rel = `${b.link.replace(/^\/+/, "")}.md`;
+						const title = (b.text || "New Page").trim() || "New Page";
+						try {
+							const created = t.create_file(rel, `# ${title}\n`);
+							note_write(
+								path.join(global_source_path, created),
+								`# ${title}\n`,
+							);
+						} catch (e) {
+							// An existing file is fine — the leaf just links to it.
+							if (!/already exists/i.test(String(e && e.message))) throw e;
+						}
+					}
+					t.create_leaf(b.parentId ?? null, b.index, b.text ?? "", b.link);
+					return t.to_dto();
+				});
+			});
+
+			app.post("/_edit/toc/add-section", (req, res) => {
+				const b = req.body || {};
+				toc_op(res, (t) => {
+					t.add_section(b.parentId ?? null, b.index, b.text ?? "");
+					return t.to_dto();
+				});
+			});
+
+			app.post("/_edit/toc/update", (req, res) => {
+				const b = req.body || {};
+				toc_op(res, (t) => {
+					if (typeof b.id !== "string" || !b.id) throw new Error("Missing id");
+					t.update_node(b.id, {
+						text: b.text,
+						link: b.link,
+						expand: b.expand,
+						newId: b.newId,
+					});
+					return t.to_dto();
+				});
+			});
+
+			// Remove a nav node; { deleteFile: true } also deletes its page file.
+			app.delete("/_edit/toc/:id", (req, res) => {
+				const b = req.body || {};
+				toc_op(res, (t) => {
+					const node = t.remove_node(req.params.id);
+					let fileDeleted = false;
+					if (b.deleteFile && typeof node.link === "string" && node.link) {
+						const r = t._resolve_file(node.link);
+						if (r.file && r.fileExists) {
+							t.delete_file(r.file);
+							fileDeleted = true;
+						}
+					}
+					return { tree: t.to_dto(), fileDeleted };
+				});
+			});
+
+			// Upload an image/asset (raw bytes) to a path inside the book —
+			// mirrors `hdoc edit` PUT /api/upload. resolve_upload restricts to
+			// image extensions and sandboxes inside the book folder.
+			app.put(
+				"/_edit/upload",
+				express.raw({ type: () => true, limit: "25mb" }),
+				(req, res) => {
+					toc_op(res, (t) => {
+						const { abs, rel } = t.resolve_upload(
+							String(req.query.path || ""),
+						);
+						if (!Buffer.isBuffer(req.body) || req.body.length === 0) {
+							throw new Error("Empty upload body");
+						}
+						fs.mkdirSync(path.dirname(abs), { recursive: true });
+						fs.writeFileSync(abs, req.body);
+						note_write(abs, req.body);
+						return { ok: true, file: rel, bytes: req.body.length };
+					});
+				},
+			);
+
+			// Pages/assets on disk (decoupled from nav; "_" folders excluded).
+			app.get("/_edit/files", (req, res) =>
+				toc_op(res, (t) => t.list_files()),
+			);
+
+			app.post("/_edit/files", (req, res) => {
+				const b = req.body || {};
+				toc_op(res, (t) => {
+					const content_txt =
+						typeof b.content === "string" ? b.content : "";
+					const created = t.create_file(b.path, content_txt);
+					note_write(path.join(global_source_path, created), content_txt);
+					return { created, ...t.list_files() };
+				});
+			});
+
+			app.post("/_edit/files/folder", (req, res) => {
+				const b = req.body || {};
+				toc_op(res, (t) => ({
+					created: t.create_folder(b.path),
+					...t.list_files(),
+				}));
+			});
+
+			app.delete("/_edit/files", (req, res) => {
+				const b = req.body || {};
+				toc_op(res, (t) => ({
+					deleted: t.delete_file(b.path),
+					...t.list_files(),
+				}));
+			});
+
+			// --- live filesystem sync (SSE + recursive watcher) -----------------
+			// Mirrors `hdoc edit`: connected clients get {type:'structure'} when
+			// hdocbook.json changes externally (the TOC model is reloaded first)
+			// and {type:'fs', path, exists} for other files. Our own writes are
+			// suppressed by content hash. fs.watch({recursive}) is Windows/macOS;
+			// on Linux this degrades to a console warning (same as hdoc edit).
+			const sse_clients = new Set();
+			const recent_writes = new Map(); // absPath -> { hash, time }
+
+			const note_write = (abs, data) => {
+				recent_writes.set(abs, { hash: sha1(data), time: Date.now() });
+				const cutoff = Date.now() - 10000;
+				for (const [k, v] of recent_writes) {
+					if (v.time < cutoff) recent_writes.delete(k);
+				}
+			};
+
+			const broadcast = (evt) => {
+				const data = `data: ${JSON.stringify(evt)}\n\n`;
+				for (const c of sse_clients) {
+					try {
+						c.write(data);
+					} catch {
+						/* client gone */
+					}
+				}
+			};
+
+			const hdocbook_json_abs = path.join(
+				global_source_path,
+				docId,
+				"hdocbook.json",
+			);
+
+			const handle_fs_change = (rel) => {
+				const abs = path.join(global_source_path, rel);
+				let exists = false;
+				let buf = null;
+				try {
+					if (fs.existsSync(abs) && fs.statSync(abs).isFile()) {
+						exists = true;
+						buf = fs.readFileSync(abs);
+					}
+				} catch {
+					/* transient */
+				}
+				if (exists && buf) {
+					const note = recent_writes.get(abs);
+					if (
+						note &&
+						note.hash === sha1(buf) &&
+						Date.now() - note.time < 5000
+					) {
+						return; // our own write
+					}
+				}
+				if (abs === hdocbook_json_abs) {
+					if (exists && toc) {
+						try {
+							toc.reload();
+						} catch (e) {
+							console.error("TOC reload failed:", e.message);
+						}
+					}
+					broadcast({ type: "structure" });
+					return;
+				}
+				broadcast({ type: "fs", path: rel, exists });
+			};
+
+			const watch_debounce = new Map();
+			try {
+				const watch_root = path.join(global_source_path, docId);
+				fs.watch(watch_root, { recursive: true }, (_event, filename) => {
+					if (!filename) return;
+					const rel = `${docId}/${String(filename).split(path.sep).join("/")}`;
+					clearTimeout(watch_debounce.get(rel));
+					watch_debounce.set(
+						rel,
+						setTimeout(() => {
+							watch_debounce.delete(rel);
+							handle_fs_change(rel);
+						}, 120),
+					);
+				});
+			} catch (e) {
+				console.error("File watching unavailable:", e.message);
+			}
+
+			app.get("/_edit/events", (req, res) => {
+				res.writeHead(200, {
+					"Content-Type": "text/event-stream",
+					// no-transform: the compression middleware otherwise buffers
+					// the stream and events never reach the client
+					"Cache-Control": "no-cache, no-transform",
+					Connection: "keep-alive",
+				});
+				res.write(": connected\n\n");
+				sse_clients.add(res);
+				const hb = setInterval(() => {
+					try {
+						res.write(": hb\n\n");
+					} catch {
+						/* ignore */
+					}
+				}, 25000);
+				req.on("close", () => {
+					clearInterval(hb);
+					sse_clients.delete(res);
+				});
+			});
+
+			// --- custom spellcheck dictionary -----------------------------------
+			// Persists into hdocbook-project.json validation.spellcheckDictionary
+			// (same store `hdoc edit` and `hdoc validate` use), preserving the
+			// file's indent style. The inline lint reads the project file fresh
+			// on every pass, so the word stops being flagged immediately.
+			app.get("/_edit/dictionary", (req, res) => {
+				const proj = fresh_project();
+				const v = proj && proj.validation;
+				res.json({
+					words:
+						v && Array.isArray(v.spellcheckDictionary)
+							? v.spellcheckDictionary
+							: [],
+				});
+			});
+
+			app.post("/_edit/dictionary", (req, res) => {
+				const word = String((req.body || {}).word || "").trim();
+				if (!word) return res.status(400).json({ error: "Missing word" });
+				try {
+					const proj_path = path.join(
+						global_source_path,
+						"hdocbook-project.json",
+					);
+					const raw = fs.readFileSync(proj_path, "utf8");
+					const m = raw.match(/\n(\t+| +)\S/);
+					const indent = m ? (m[1].includes("\t") ? "\t" : m[1].length) : 2;
+					const proj = JSON.parse(raw);
+					if (!proj.validation) proj.validation = {};
+					if (!Array.isArray(proj.validation.spellcheckDictionary)) {
+						proj.validation.spellcheckDictionary = [];
+					}
+					const list = proj.validation.spellcheckDictionary;
+					if (!list.some((x) => x.toLowerCase() === word.toLowerCase())) {
+						list.push(word);
+						list.sort((a, b) => a.localeCompare(b));
+					}
+					fs.writeFileSync(
+						proj_path,
+						JSON.stringify(proj, null, indent) +
+							(raw.endsWith("\n") ? "\n" : ""),
+						"utf8",
+					);
+					res.json({ ok: true, words: list });
+				} catch (e) {
+					res.status(400).json({ error: String((e && e.message) || e) });
+				}
+			});
+
+			// --- AI writing assistance (Anthropic Messages API, native fetch) ---
+			// Same contract as `hdoc edit` /api/assist: streams the edited
+			// Markdown back as plain text; key from ANTHROPIC_API_KEY (never
+			// reaches the browser); model/limits from hdocbook-project.json
+			// "ai" — read fresh per request rather than at startup.
+			const AI_SYSTEM = [
+				"You are a writing assistant embedded in the Hornbill HDocBook editor.",
+				"You help authors edit technical documentation written in Markdown.",
+				"",
+				"House style:",
+				"- Use US / International English spelling and vocabulary.",
+				"- Be clear, concise and direct; prefer active voice and present tense.",
+				"- Preserve the author's meaning and intent. Never invent facts, features or details.",
+				"",
+				"Markdown and HDocBook rules:",
+				"- Preserve all Markdown structure: headings, lists, tables, links, images, code fences and inline `code`.",
+				"- Never change the contents of fenced code blocks or inline code spans.",
+				"- Preserve HDocBook markup verbatim, including [[INCLUDE ...]], [[File:...]] and wiki-style directives.",
+				"- Do not change link or image URLs.",
+				"",
+				"Output rules:",
+				"- Return only the edited Markdown for the supplied text — no preamble, explanation, commentary or surrounding code fence.",
+			].join("\n");
+
+			const AI_ACTIONS = {
+				rewrite:
+					"Rewrite the supplied text to improve clarity, flow and readability while preserving its meaning and all Markdown structure.",
+				grammar:
+					"Correct only grammar, spelling and punctuation in the supplied text. Make the minimum changes necessary; do not otherwise rephrase or restructure it.",
+				tighten:
+					"Make the supplied text more concise and direct without losing information. Remove redundancy and wordiness.",
+			};
+
+			app.post("/_edit/assist", async (req, res) => {
+				const apiKey = process.env.ANTHROPIC_API_KEY;
+				if (!apiKey) {
+					return res.status(503).json({
+						error:
+							"ANTHROPIC_API_KEY is not set. Set it in the environment running `hdoc serve` to enable AI writing assistance.",
+					});
+				}
+
+				const body = req.body || {};
+				const text = typeof body.text === "string" ? body.text : "";
+				const action =
+					typeof body.action === "string" ? body.action : "rewrite";
+				const instruction =
+					typeof body.instruction === "string"
+						? body.instruction.trim()
+						: "";
+				if (!text.trim()) {
+					return res.status(400).json({ error: "No text to edit" });
+				}
+
+				const proj = fresh_project();
+				const ai_cfg =
+					proj && proj.ai && typeof proj.ai === "object" ? proj.ai : {};
+				const model =
+					typeof ai_cfg.model === "string" && ai_cfg.model
+						? ai_cfg.model
+						: "claude-sonnet-4-6";
+				const max_tokens = Number.isFinite(ai_cfg.maxTokens)
+					? ai_cfg.maxTokens
+					: 4096;
+
+				const task =
+					action === "custom"
+						? instruction || "Improve the supplied text."
+						: AI_ACTIONS[action] || AI_ACTIONS.rewrite;
+
+				let system = AI_SYSTEM;
+				const dict =
+					proj &&
+					proj.validation &&
+					Array.isArray(proj.validation.spellcheckDictionary)
+						? proj.validation.spellcheckDictionary
+						: [];
+				if (dict.length) {
+					system += `\n\nThe following are correctly-spelled domain terms; keep them as-is: ${dict.join(", ")}.`;
+				}
+
+				const user_text = `Task: ${task}\n\nHere is the Markdown to edit:\n\n<text>\n${text}\n</text>\n\nReturn only the edited Markdown.`;
+
+				let upstream;
+				try {
+					upstream = await fetch("https://api.anthropic.com/v1/messages", {
+						method: "POST",
+						headers: {
+							"content-type": "application/json",
+							"x-api-key": apiKey,
+							"anthropic-version": "2023-06-01",
+						},
+						body: JSON.stringify({
+							model,
+							max_tokens,
+							stream: true,
+							system: [
+								{
+									type: "text",
+									text: system,
+									cache_control: { type: "ephemeral" },
+								},
+							],
+							messages: [{ role: "user", content: user_text }],
+						}),
+					});
+				} catch (e) {
+					return res.status(502).json({
+						error: `Failed to reach the Anthropic API: ${String((e && e.message) || e)}`,
+					});
+				}
+
+				if (!upstream.ok || !upstream.body) {
+					let detail = `Anthropic API error (HTTP ${upstream.status})`;
+					try {
+						const err = await upstream.json();
+						if (err && err.error && err.error.message) {
+							detail = err.error.message;
+						}
+					} catch {
+						/* non-JSON error body */
+					}
+					return res
+						.status(upstream.status === 401 ? 401 : 502)
+						.json({ error: detail });
+				}
+
+				res.writeHead(200, {
+					"Content-Type": "text/plain; charset=utf-8",
+					"Cache-Control": "no-cache",
+				});
+				const reader = upstream.body.getReader();
+				const decoder = new TextDecoder();
+				let buf = "";
+				try {
+					for (;;) {
+						const { done, value } = await reader.read();
+						if (done) break;
+						buf += decoder.decode(value, { stream: true });
+						let nl;
+						while ((nl = buf.indexOf("\n")) >= 0) {
+							const line = buf.slice(0, nl).trim();
+							buf = buf.slice(nl + 1);
+							if (!line.startsWith("data:")) continue;
+							const payload = line.slice(5).trim();
+							if (!payload) continue;
+							try {
+								const evt = JSON.parse(payload);
+								if (
+									evt.type === "content_block_delta" &&
+									evt.delta &&
+									evt.delta.type === "text_delta"
+								) {
+									res.write(evt.delta.text);
+								}
+							} catch {
+								/* partial frame split across chunks */
+							}
+						}
+					}
+				} catch (e) {
+					console.error("AI assist stream error:", (e && e.message) || e);
+				}
+				res.end();
+			});
+
+			// --- validation ---------------------------------------------------
+			//
+			// Three tiers:
+			//   nav  — fast in-process checks over the navigation tree
+			//   page — fast in-process checks over one article (md + rendered
+			//          html), optional external-link HTTP checks
+			//   book — the REAL `hdoc validate`, spawned as a child process
+			//          (full build pipeline; wipes/rebuilds _work). One at a
+			//          time; may run for minutes on a large book.
+
+			const fresh_project = () => {
+				try {
+					return JSON.parse(
+						fs.readFileSync(
+							path.join(global_source_path, "hdocbook-project.json"),
+							"utf8",
+						),
+					);
+				} catch {
+					return hdocbook_project;
+				}
+			};
+
+			app.post("/_edit/validate/nav", (req, res) => {
+				toc_op(res, (t) => {
+					t.reload();
+					return {
+						nodes: serve_validate.validate_nav(
+							t.items,
+							global_source_path,
+							docId,
+							fresh_project(),
+						),
+					};
+				});
+			});
+
+			app.post("/_edit/validate/page", async (req, res) => {
+				const body = req.body || {};
+				const resolved = resolve_page_md(body.path);
+				if (!resolved) {
+					return res
+						.status(404)
+						.json({ error: "No markdown source found for this page" });
+				}
+				try {
+					const md = fs.readFileSync(resolved.abs, "utf8");
+					const html = await content.render_page(
+						resolved.abs,
+						md,
+						resolved.logical,
+					);
+					const result = await serve_validate.validate_page({
+						source_path: global_source_path,
+						docId,
+						rel: resolved.rel,
+						logical: resolved.logical,
+						md,
+						html,
+						project: fresh_project(),
+						external: body.external !== false,
+					});
+					res.json({ file: resolved.rel, ...result });
+				} catch (e) {
+					res.status(500).json({ error: String((e && e.message) || e) });
+				}
+			});
+
+			let validate_child = null;
+			app.post("/_edit/validate/book", (req, res) => {
+				if (validate_child) {
+					return res
+						.status(409)
+						.json({ error: "A validation run is already in progress" });
+				}
+				const body = req.body || {};
+				const args = [path.join(__dirname, "hdoc.js"), "validate", "--quiet"];
+				if (body.externalLinks === false) args.push("--no-links");
+				const child = spawn(process.execPath, args, {
+					cwd: global_source_path,
+					windowsHide: true,
+					env: process.env,
+				});
+				validate_child = child;
+				let out = "";
+				const cap = (buf) => {
+					out += String(buf);
+					// keep the tail — validation output is what matters and a huge
+					// build log would bloat the response
+					if (out.length > 512 * 1024) out = out.slice(-512 * 1024);
+				};
+				child.stdout.on("data", cap);
+				child.stderr.on("data", cap);
+				const timer = setTimeout(
+					() => {
+						cap("\n[Timed out after 10 minutes — killed]");
+						child.kill("SIGKILL");
+					},
+					10 * 60 * 1000,
+				);
+				child.on("close", (code) => {
+					clearTimeout(timer);
+					validate_child = null;
+					// strip ANSI color codes and spinner control chars
+					// biome-ignore lint/suspicious/noControlCharactersInRegex: ANSI stripping
+					const clean = out
+						.replace(/\[[0-9;]*m/g, "")
+						.replace(/[\r][^\n]*\r/g, "");
+					res.json({ code, output: clean });
+				});
+				child.on("error", (e) => {
+					clearTimeout(timer);
+					validate_child = null;
+					res.status(500).json({ error: String((e && e.message) || e) });
+				});
+			});
+
 			// Render an UNSAVED buffer to the final page fragment (document
 			// header included) through the shared pipeline, for live preview.
 			app.post("/_edit/preview", async (req, res) => {
@@ -234,13 +869,35 @@
 						body.content,
 						resolved.logical,
 					);
-					res.json({ html });
+					// inline lint rides the same debounced round trip: British
+					// spellings + syntactic/local link problems as editor
+					// underline ranges (no HTTP checks here)
+					let findings = [];
+					try {
+						findings = serve_validate.lint_markdown({
+							md: body.content,
+							source_path: global_source_path,
+							docId,
+							project: fresh_project(),
+							logical: resolved.logical,
+						});
+					} catch (e) {
+						console.error("Inline lint failed:", (e && e.message) || e);
+					}
+					res.json({ html, findings });
 				} catch (e) {
 					res.status(500).json({ error: String((e && e.message) || e) });
 				}
 			});
 		}
 
+		// en-US dictionary for the typo spell-check worker
+		// (ui/js/hdoc-edit-spell.js) — same files `hdoc edit` serves.
+		app.use(
+			"/dict",
+			express.static(path.join(__dirname, "editor", "dist", "dict")),
+		);
+
 		// Local preview serves exactly one book — skip the viewer's library home
 		// and land straight in the book.
 		app.get("/", (req, res) => {

package/hdoc-toc.js

@@ -118,12 +118,17 @@ class TocModel {
 	}
 
 	// Resolve a leaf link (path without extension) to a source-relative file.
+	// Links may carry a leading slash (both "/docId/page" and "docId/page"
+	// appear in real books) — the returned rel is normalized to the
+	// slash-free, forward-slash form list_files() produces, or the linked
+	// flags and file labels built from it never match.
 	_resolve_file(link) {
-		const rel = `${link}.md`;
+		const clean = String(link).replace(/^\/+/, "");
+		const rel = `${clean}.md`;
 		if (fs.existsSync(path.join(this.source_path, rel))) {
 			return { file: rel, fileExists: true };
 		}
-		const rel_index = path.join(link, "index.md");
+		const rel_index = `${clean}/index.md`;
 		if (fs.existsSync(path.join(this.source_path, rel_index))) {
 			return { file: rel_index, fileExists: true };
 		}