@waveso/docs 0.4.0 → 0.5.0

package/dist/source.js

@@ -1,6 +1,7 @@
 import { docsError } from "./docs-error.js";
 import { parseFrontmatter } from "./frontmatter.js";
 import { orderNavEntries, readDocsMeta } from "./meta.js";
+import { createSemaphore } from "./semaphore.js";
 import { encodeSegments, toAliasRoute } from "./route-path.js";
 import { readFile, readdir, realpath, stat } from "node:fs/promises";
 import path from "node:path";
@@ -12,6 +13,48 @@ const PAGE_EXTENSION = ".md";
 /** A directory whose `index.md` is the directory's own route. */
 const INDEX_NAME = "index";
 /**
+* Filesystem calls in flight across the whole process, at most.
+*
+* ⚠️ THE SCAN USED TO OPEN EVERY MARKDOWN FILE AT ONCE. `scanDir` recursed into
+* its subdirectories in parallel and read that directory's pages with a bare
+* `Promise.all`, so the number of `readFile` calls in flight equalled the number
+* of markdown files in the entire tree. On a 1,200-page corpus and the common
+* 1,024-descriptor soft limit, `next build` died with a bare `EMFILE: too many
+* open files` — no error code, no mention that this was the docs scan, and
+* nothing pointing at the fix. Exactly the large content set this package is
+* for, and the failure got worse as the site grew.
+*
+* 64 is far under every default soft limit — 1,024 on Linux and CI images, 256
+* on an unconfigured macOS shell — so the descriptors stop scaling with the
+* corpus while staying safe on the tightest of those.
+*
+* ⚠️ IT IS NOT FREE, AND THE COST IS NOT THE CONCURRENCY. Measured over 1,201
+* pages in 49 directories, nine runs, medians: **88 ms ungated, 106 ms at a
+* bound of 16, 102 ms at 64, 101 ms at 128 and at 256.** Flat from 64 upwards,
+* which says the ~14 ms is the gate's own per-call overhead — about 3,600
+* `run` closures — and not the reduced parallelism. libuv's filesystem pool is
+* four threads by default, so there was never 1,201-way parallelism to lose.
+*
+* So the number is chosen for the tightest descriptor limit rather than for
+* speed, because above 64 there is no speed left to buy. 14 ms sits against a
+* build that highlights those same 1,201 pages with Shiki, which is three
+* orders of magnitude more.
+*/
+const SCAN_CONCURRENCY = 64;
+/**
+* The one gate, for the whole process.
+*
+* Module scope rather than per-scan, because descriptors are a process resource:
+* two routes scanning two content directories at once would each stay under a
+* per-scan bound and together exceed the only one that matters.
+*
+* ⚠️ LEAF CALLS ONLY — NEVER THE RECURSION. Gating `scanDir` itself deadlocks as
+* soon as every slot is held by a directory waiting for a slot to read its
+* children. Bounding the `readFile`/`readdir`/`stat`/`realpath` calls bounds the
+* descriptors exactly, and leaves the walk as parallel as it was.
+*/
+const fsGate = createSemaphore(SCAN_CONCURRENCY);
+/**
 * Apply {@link DocsConfig} defaults and resolve `contentDir` against
 * `process.cwd()` — the project root under both `next build` and `vite build`.
 *
@@ -85,7 +128,7 @@ function buildSource(config) {
 	const load = () => {
 		cached ??= scan(config).catch((err) => {
 			cached = null;
-			throw err;
+			throw describeScanFailure(err, config.contentDir);
 		});
 		return cached;
 	};
@@ -143,9 +186,32 @@ function buildSource(config) {
 		}
 	};
 }
+/**
+* Say that a descriptor exhaustion happened here, and that it was not us.
+*
+* `EMFILE`/`ENFILE` arrive from Node as a bare `Error` with no hint of what was
+* being read, and the message a reader used to get was
+* `EMFILE: too many open files, open '<contentDir>/p829.md'` and nothing else.
+* That is a filename out of a corpus of a thousand, from a stack of `dist/`
+* frames, during a `next build` that mentions no package.
+*
+* The scan itself is bounded at {@link SCAN_CONCURRENCY} now, so reaching this
+* means the limit is lower than that or something else in the process has the
+* descriptors — which is the useful thing to be told, and the opposite of what a
+* reader concludes from an error naming one of their own markdown files.
+*
+* Only these two codes, and only when the failure is not already ours: a
+* `broken-symlink` or an `invalid-frontmatter` that came out of the scan is
+* already a better message than anything this could add.
+*/
+function describeScanFailure(err, contentDir) {
+	const code = err?.code;
+	if (code !== "EMFILE" && code !== "ENFILE") return err;
+	return docsError("descriptor-limit", `the process ran out of file descriptors while scanning ${contentDir}. The scan holds at most ${String(SCAN_CONCURRENCY)} open at a time, so something else has them — raise the limit (\`ulimit -n\`, or \`LimitNOFILE\` under systemd) rather than shrinking the corpus. If it persists with a limit well above that, it is a bug in this package: https://github.com/wavedotso/wave-docs/issues`, { cause: err });
+}
 async function scan(config) {
 	await assertContentDir(config.contentDir);
-	const root = await scanDir(config.contentDir, [], "", config, /* @__PURE__ */ new Set([await realpath(config.contentDir)]));
+	const root = await scanDir(config.contentDir, [], "", config, /* @__PURE__ */ new Set([await fsGate.run(() => realpath(config.contentDir))]));
 	const files = [];
 	const bySlug = /* @__PURE__ */ new Map();
 	collect(root, files, bySlug);
@@ -157,13 +223,13 @@ async function scan(config) {
 }
 async function assertContentDir(contentDir) {
 	try {
-		if ((await stat(contentDir)).isDirectory()) return;
+		if ((await fsGate.run(() => stat(contentDir))).isDirectory()) return;
 	} catch {}
 	throw docsError("missing-content-dir", `Docs content directory not found: ${contentDir}\nSet \`contentDir\` to a directory of markdown files; relative paths resolve against the working directory (${process.cwd()}).`);
 }
 const SKIP = { kind: "skip" };
 async function scanDir(absPath, segments, name, config, ancestors) {
-	const [meta, entries] = await Promise.all([readDocsMeta(absPath), readdir(absPath, { withFileTypes: true })]);
+	const [meta, entries] = await Promise.all([fsGate.run(() => readDocsMeta(absPath)), fsGate.run(() => readdir(absPath, { withFileTypes: true }))]);
 	const classified = await Promise.all(entries.map((entry) => ({
 		entry,
 		name: entry.name.normalize("NFC")
@@ -206,7 +272,7 @@ async function classifyEntry(dirPath, entry, name) {
 		if (!entry.isSymbolicLink()) return SKIP;
 		let target;
 		try {
-			target = await stat(absPath);
+			target = await fsGate.run(() => stat(absPath));
 		} catch {
 			if (!isPageFile(name)) return SKIP;
 			throw docsError("broken-symlink", `@waveso/docs: ${absPath} is a broken symbolic link, and it names a markdown page — so skipping it would delete a route nobody asked to delete. Point it at an existing file, or remove the link.`);
@@ -223,20 +289,23 @@ async function classifyEntry(dirPath, entry, name) {
 		kind: "dir",
 		absPath,
 		name,
-		realPath: await realpath(absPath)
+		realPath: await fsGate.run(() => realpath(absPath))
 	};
 	return SKIP;
 }
 async function readPage(filePath, name, dirSegments, config) {
-	const raw = await readFile(filePath, "utf8");
+	const raw = await fsGate.run(() => readFile(filePath, "utf8"));
 	const relativePath = toPosix(path.relative(config.contentDir, filePath));
 	let data;
 	let content;
+	let frontmatterLines = 0;
 	try {
-		const file = new VFile({ value: raw.charCodeAt(0) === 65279 ? raw.slice(1) : raw });
+		const source = raw.charCodeAt(0) === 65279 ? raw.slice(1) : raw;
+		const file = new VFile({ value: source });
 		matter(file, { strip: true });
 		data = file.data.matter;
 		content = String(file);
+		if (source.endsWith(content)) frontmatterLines = source.slice(0, source.length - content.length).split("\n").length - 1;
 	} catch (err) {
 		const reason = err instanceof Error ? err.message : String(err);
 		throw docsError("invalid-frontmatter", `Could not parse the frontmatter block in ${relativePath}: ${reason}`, { cause: err });
@@ -253,7 +322,8 @@ async function readPage(filePath, name, dirSegments, config) {
 			filePath,
 			relativePath,
 			frontmatter,
-			content
+			content,
+			frontmatterLines
 		}
 	};
 }
@@ -409,7 +479,7 @@ function toHref(basePath, segments) {
 	return `${basePath}/${encodeSegments(segments)}`;
 }
 function normalizeBasePath(basePath) {
-	const trimmed = basePath.trim().replace(/\/+$/, "");
+	const trimmed = basePath.trim().replace(/\/{2,}/g, "/").replace(/\/+$/, "");
 	if (trimmed === "") return "";
 	return trimmed.startsWith("/") ? trimmed : `/${trimmed}`;
 }

package/dist/types.d.ts

@@ -67,6 +67,24 @@ interface DocFile<TFrontmatter extends DocFrontmatter = DocFrontmatter> {
   frontmatter: TFrontmatter;
   /** Markdown body with the frontmatter block removed. */
   content: string;
+  /**
+   * How many lines the frontmatter block took up, so an error can name the line
+   * in the file rather than in the body.
+   *
+   * ⚠️ WITHOUT IT, EVERY LINK ERROR POINTED INTO THE FRONTMATTER. `content` has
+   * the block removed, so remark counts `node.position.start.line` from the
+   * first line of the *body* — and `broken-link`, `draft-link` and `alias-link`
+   * all report `relativePath:line`, the exact `file:line` form a terminal and an
+   * editor turn into a jump. A page with four frontmatter fields is six lines
+   * out: a link on line 10 was reported at line 4, which is the middle of the
+   * block that was deleted.
+   *
+   * Set by `@waveso/docs/source`. Optional and treated as `0` when absent,
+   * because a host loading content itself — the documented reason `./render` is
+   * an entry point — hands over a body with no block in front of it and has
+   * nothing to offset.
+   */
+  frontmatterLines?: number | undefined;
 }
 /** A link to a documentation page. */
 interface DocNavPage {
@@ -257,6 +275,18 @@ href: string, from: DocLinkContext) => string | undefined;
  * blindly prefixes what it is handed produces `/cdn//logo.png` and
  * `/cdn/https://example.com/a.png`. Branch on them, or return `undefined` to
  * leave the src as authored.
+ *
+ * ⚠️ IT IS A FILE PATH, NOT AN HREF. `src` is percent-decoded, so
+ * `./getting%20started.png` — what GitHub's editor writes when you drag in a
+ * file whose name has a space — arrives as `getting started.png`, the name that
+ * is actually on disk. Any `?query` and `#fragment` are split off before the
+ * call and re-attached to whatever you return, so `./diagram.png?v=2` arrives
+ * as `diagram.png` and the reader still gets the `?v=2`. Return a src carrying
+ * a query or a fragment of your own and yours is kept instead, because two `?`
+ * in one URL is not a URL.
+ *
+ * All three used to be handed over raw, and `readFile(join(dir, src))` — the
+ * implementation the README gives — threw `ENOENT` on every one of them.
  */
 type ImageResolver = (src: string, from: DocLinkContext) => Promise<{
   src: string;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@waveso/docs",
-  "version": "0.4.0",
+  "version": "0.5.0",
   "description": "Zero parser bytes in the browser: markdown docs for Next.js, built to hast in Node and rendered as your components",
   "type": "module",
   "sideEffects": [