@waveso/docs 0.4.0 → 0.6.0

package/dist/source.js

@@ -1,6 +1,7 @@
 import { docsError } from "./docs-error.js";
 import { parseFrontmatter } from "./frontmatter.js";
 import { orderNavEntries, readDocsMeta } from "./meta.js";
+import { createSemaphore } from "./semaphore.js";
 import { encodeSegments, toAliasRoute } from "./route-path.js";
 import { readFile, readdir, realpath, stat } from "node:fs/promises";
 import path from "node:path";
@@ -12,6 +13,48 @@ const PAGE_EXTENSION = ".md";
 /** A directory whose `index.md` is the directory's own route. */
 const INDEX_NAME = "index";
 /**
+* Filesystem calls in flight across the whole process, at most.
+*
+* ⚠️ THE SCAN USED TO OPEN EVERY MARKDOWN FILE AT ONCE. `scanDir` recursed into
+* its subdirectories in parallel and read that directory's pages with a bare
+* `Promise.all`, so the number of `readFile` calls in flight equalled the number
+* of markdown files in the entire tree. On a 1,200-page corpus and the common
+* 1,024-descriptor soft limit, `next build` died with a bare `EMFILE: too many
+* open files` — no error code, no mention that this was the docs scan, and
+* nothing pointing at the fix. Exactly the large content set this package is
+* for, and the failure got worse as the site grew.
+*
+* 64 is far under every default soft limit — 1,024 on Linux and CI images, 256
+* on an unconfigured macOS shell — so the descriptors stop scaling with the
+* corpus while staying safe on the tightest of those.
+*
+* ⚠️ IT IS NOT FREE, AND THE COST IS NOT THE CONCURRENCY. Measured over 1,201
+* pages in 49 directories, nine runs, medians: **88 ms ungated, 106 ms at a
+* bound of 16, 102 ms at 64, 101 ms at 128 and at 256.** Flat from 64 upwards,
+* which says the ~14 ms is the gate's own per-call overhead — about 3,600
+* `run` closures — and not the reduced parallelism. libuv's filesystem pool is
+* four threads by default, so there was never 1,201-way parallelism to lose.
+*
+* So the number is chosen for the tightest descriptor limit rather than for
+* speed, because above 64 there is no speed left to buy. 14 ms sits against a
+* build that highlights those same 1,201 pages with Shiki, which is three
+* orders of magnitude more.
+*/
+const SCAN_CONCURRENCY = 64;
+/**
+* The one gate, for the whole process.
+*
+* Module scope rather than per-scan, because descriptors are a process resource:
+* two routes scanning two content directories at once would each stay under a
+* per-scan bound and together exceed the only one that matters.
+*
+* ⚠️ LEAF CALLS ONLY — NEVER THE RECURSION. Gating `scanDir` itself deadlocks as
+* soon as every slot is held by a directory waiting for a slot to read its
+* children. Bounding the `readFile`/`readdir`/`stat`/`realpath` calls bounds the
+* descriptors exactly, and leaves the walk as parallel as it was.
+*/
+const fsGate = createSemaphore(SCAN_CONCURRENCY);
+/**
 * Apply {@link DocsConfig} defaults and resolve `contentDir` against
 * `process.cwd()` — the project root under both `next build` and `vite build`.
 *
@@ -28,7 +71,9 @@ function resolveDocsConfig(config) {
 		),
 		basePath: normalizeBasePath(config.basePath ?? "/docs"),
 		includeDrafts: config.includeDrafts ?? false,
-		assertLinks: config.assertLinks ?? true,
+		onBrokenLinks: config.onBrokenLinks ?? "throw",
+		onBrokenAnchors: config.onBrokenAnchors ?? "throw",
+		externalRoutes: config.externalRoutes ?? [],
 		...config.frontmatterSchema === void 0 ? {} : { frontmatterSchema: config.frontmatterSchema }
 	};
 }
@@ -71,7 +116,9 @@ function createDocsSource(config) {
 		resolved.contentDir,
 		resolved.basePath,
 		resolved.includeDrafts,
-		resolved.assertLinks,
+		resolved.onBrokenLinks,
+		resolved.onBrokenAnchors,
+		resolved.externalRoutes.join(","),
 		schemaKey(resolved.frontmatterSchema)
 	].join("\0");
 	const existing = sources.get(key);
@@ -85,7 +132,7 @@ function buildSource(config) {
 	const load = () => {
 		cached ??= scan(config).catch((err) => {
 			cached = null;
-			throw err;
+			throw describeScanFailure(err, config.contentDir);
 		});
 		return cached;
 	};
@@ -143,9 +190,32 @@ function buildSource(config) {
 		}
 	};
 }
+/**
+* Say that a descriptor exhaustion happened here, and that it was not us.
+*
+* `EMFILE`/`ENFILE` arrive from Node as a bare `Error` with no hint of what was
+* being read, and the message a reader used to get was
+* `EMFILE: too many open files, open '<contentDir>/p829.md'` and nothing else.
+* That is a filename out of a corpus of a thousand, from a stack of `dist/`
+* frames, during a `next build` that mentions no package.
+*
+* The scan itself is bounded at {@link SCAN_CONCURRENCY} now, so reaching this
+* means the limit is lower than that or something else in the process has the
+* descriptors — which is the useful thing to be told, and the opposite of what a
+* reader concludes from an error naming one of their own markdown files.
+*
+* Only these two codes, and only when the failure is not already ours: a
+* `broken-symlink` or an `invalid-frontmatter` that came out of the scan is
+* already a better message than anything this could add.
+*/
+function describeScanFailure(err, contentDir) {
+	const code = err?.code;
+	if (code !== "EMFILE" && code !== "ENFILE") return err;
+	return docsError("descriptor-limit", `the process ran out of file descriptors while scanning ${contentDir}. The scan holds at most ${String(SCAN_CONCURRENCY)} open at a time, so something else has them — raise the limit (\`ulimit -n\`, or \`LimitNOFILE\` under systemd) rather than shrinking the corpus. If it persists with a limit well above that, it is a bug in this package: https://github.com/wavedotso/wave-docs/issues`, { cause: err });
+}
 async function scan(config) {
 	await assertContentDir(config.contentDir);
-	const root = await scanDir(config.contentDir, [], "", config, /* @__PURE__ */ new Set([await realpath(config.contentDir)]));
+	const root = await scanDir(config.contentDir, [], "", config, /* @__PURE__ */ new Set([await fsGate.run(() => realpath(config.contentDir))]));
 	const files = [];
 	const bySlug = /* @__PURE__ */ new Map();
 	collect(root, files, bySlug);
@@ -157,13 +227,13 @@ async function scan(config) {
 }
 async function assertContentDir(contentDir) {
 	try {
-		if ((await stat(contentDir)).isDirectory()) return;
+		if ((await fsGate.run(() => stat(contentDir))).isDirectory()) return;
 	} catch {}
 	throw docsError("missing-content-dir", `Docs content directory not found: ${contentDir}\nSet \`contentDir\` to a directory of markdown files; relative paths resolve against the working directory (${process.cwd()}).`);
 }
 const SKIP = { kind: "skip" };
 async function scanDir(absPath, segments, name, config, ancestors) {
-	const [meta, entries] = await Promise.all([readDocsMeta(absPath), readdir(absPath, { withFileTypes: true })]);
+	const [meta, entries] = await Promise.all([fsGate.run(() => readDocsMeta(absPath)), fsGate.run(() => readdir(absPath, { withFileTypes: true }))]);
 	const classified = await Promise.all(entries.map((entry) => ({
 		entry,
 		name: entry.name.normalize("NFC")
@@ -206,7 +276,7 @@ async function classifyEntry(dirPath, entry, name) {
 		if (!entry.isSymbolicLink()) return SKIP;
 		let target;
 		try {
-			target = await stat(absPath);
+			target = await fsGate.run(() => stat(absPath));
 		} catch {
 			if (!isPageFile(name)) return SKIP;
 			throw docsError("broken-symlink", `@waveso/docs: ${absPath} is a broken symbolic link, and it names a markdown page — so skipping it would delete a route nobody asked to delete. Point it at an existing file, or remove the link.`);
@@ -223,20 +293,23 @@ async function classifyEntry(dirPath, entry, name) {
 		kind: "dir",
 		absPath,
 		name,
-		realPath: await realpath(absPath)
+		realPath: await fsGate.run(() => realpath(absPath))
 	};
 	return SKIP;
 }
 async function readPage(filePath, name, dirSegments, config) {
-	const raw = await readFile(filePath, "utf8");
+	const raw = await fsGate.run(() => readFile(filePath, "utf8"));
 	const relativePath = toPosix(path.relative(config.contentDir, filePath));
 	let data;
 	let content;
+	let frontmatterLines = 0;
 	try {
-		const file = new VFile({ value: raw.charCodeAt(0) === 65279 ? raw.slice(1) : raw });
+		const source = raw.charCodeAt(0) === 65279 ? raw.slice(1) : raw;
+		const file = new VFile({ value: source });
 		matter(file, { strip: true });
 		data = file.data.matter;
 		content = String(file);
+		if (source.endsWith(content)) frontmatterLines = source.slice(0, source.length - content.length).split("\n").length - 1;
 	} catch (err) {
 		const reason = err instanceof Error ? err.message : String(err);
 		throw docsError("invalid-frontmatter", `Could not parse the frontmatter block in ${relativePath}: ${reason}`, { cause: err });
@@ -253,7 +326,8 @@ async function readPage(filePath, name, dirSegments, config) {
 			filePath,
 			relativePath,
 			frontmatter,
-			content
+			content,
+			frontmatterLines
 		}
 	};
 }
@@ -409,7 +483,7 @@ function toHref(basePath, segments) {
 	return `${basePath}/${encodeSegments(segments)}`;
 }
 function normalizeBasePath(basePath) {
-	const trimmed = basePath.trim().replace(/\/+$/, "");
+	const trimmed = basePath.trim().replace(/\/{2,}/g, "/").replace(/\/+$/, "");
 	if (trimmed === "") return "";
 	return trimmed.startsWith("/") ? trimmed : `/${trimmed}`;
 }

package/dist/types.d.ts

@@ -67,6 +67,24 @@ interface DocFile<TFrontmatter extends DocFrontmatter = DocFrontmatter> {
   frontmatter: TFrontmatter;
   /** Markdown body with the frontmatter block removed. */
   content: string;
+  /**
+   * How many lines the frontmatter block took up, so an error can name the line
+   * in the file rather than in the body.
+   *
+   * ⚠️ WITHOUT IT, EVERY LINK ERROR POINTED INTO THE FRONTMATTER. `content` has
+   * the block removed, so remark counts `node.position.start.line` from the
+   * first line of the *body* — and `broken-link`, `draft-link` and `alias-link`
+   * all report `relativePath:line`, the exact `file:line` form a terminal and an
+   * editor turn into a jump. A page with four frontmatter fields is six lines
+   * out: a link on line 10 was reported at line 4, which is the middle of the
+   * block that was deleted.
+   *
+   * Set by `@waveso/docs/source`. Optional and treated as `0` when absent,
+   * because a host loading content itself — the documented reason `./render` is
+   * an entry point — hands over a body with no block in front of it and has
+   * nothing to offset.
+   */
+  frontmatterLines?: number | undefined;
 }
 /** A link to a documentation page. */
 interface DocNavPage {
@@ -226,11 +244,24 @@ interface DocLinkContext {
   /** The source path, e.g. `'api/auth.md'`. For error messages. */
   relativePath: string;
 }
+/**
+ * What a link problem should do to a build.
+ *
+ * The three Docusaurus settled on, and for the same reason: the tool cannot
+ * know how much a given site cares, and guessing produces either a build that
+ * fails on someone's legitimate URL or one that ships a dead link quietly.
+ *
+ * `'warn'` writes to `console.warn` and continues, which on a docs site is a
+ * line in a build log — useful during a migration, not a substitute for
+ * `'throw'`.
+ */
+type DocsLinkSeverity = 'throw' | 'warn' | 'ignore';
 /**
  * Resolve an internal markdown link target to a route.
  *
  * Called for every relative link found in the source. Returning `undefined`
- * signals "not a documentation page", which — with `assertLinks` on — fails
+ * signals "not a documentation page", which — under the default
+ * `onBrokenLinks: 'throw'` — fails
  * the build rather than shipping a 404 that was valid on GitHub.
  */
 type LinkResolver = (
@@ -257,6 +288,18 @@ href: string, from: DocLinkContext) => string | undefined;
  * blindly prefixes what it is handed produces `/cdn//logo.png` and
  * `/cdn/https://example.com/a.png`. Branch on them, or return `undefined` to
  * leave the src as authored.
+ *
+ * ⚠️ IT IS A FILE PATH, NOT AN HREF. `src` is percent-decoded, so
+ * `./getting%20started.png` — what GitHub's editor writes when you drag in a
+ * file whose name has a space — arrives as `getting started.png`, the name that
+ * is actually on disk. Any `?query` and `#fragment` are split off before the
+ * call and re-attached to whatever you return, so `./diagram.png?v=2` arrives
+ * as `diagram.png` and the reader still gets the `?v=2`. Return a src carrying
+ * a query or a fragment of your own and yours is kept instead, because two `?`
+ * in one URL is not a URL.
+ *
+ * All three used to be handed over raw, and `readFile(join(dir, src))` — the
+ * implementation the README gives — threw `ENOENT` on every one of them.
  */
 type ImageResolver = (src: string, from: DocLinkContext) => Promise<{
   src: string;
@@ -291,10 +334,61 @@ interface DocsConfig<TFrontmatter extends DocFrontmatter = DocFrontmatter> {
    */
   includeDrafts?: boolean | undefined;
   /**
-   * Fail the build when an internal link resolves to a page that does not
-   * exist. Defaults to `true`; there is no good reason to turn it off.
+   * What to do about an internal link that resolves to no published page.
+   * Defaults to `'throw'`.
+   *
+   * A link that 404s was valid in the editor and on GitHub, so it is the kind
+   * of mistake nobody finds by reading. Throwing is the default for that
+   * reason, and there is rarely a good reason to lower it — `'warn'` exists
+   * for a migration where the corpus is knowingly incomplete for a while.
+   *
+   * The error names the file and the line, and offers the closest published
+   * route when the link looks like a typo of one.
+   */
+  onBrokenLinks?: DocsLinkSeverity | undefined;
+  /**
+   * What to do about a `#fragment` that no heading on the target page owns.
+   * Defaults to `'throw'`.
+   *
+   * ⚠️ THE MORE COMMON OF THE TWO LINK FAILURES, AND IT WENT UNCHECKED. A route
+   * was verified and its fragment discarded, so `[setup](./install.md#setup)`
+   * built green with no `#setup` anywhere on the page. Headings get renamed
+   * constantly and nothing renames the links into them, which is exactly why it
+   * is worth checking and exactly why it breaks.
+   *
+   * Checked against every `id` in the rendered page rather than against the
+   * table of contents, which captures `h2`–`h3` only — so a link to an `h4` is
+   * fine, and so is one to an id a `rehypePlugins` entry put on something that
+   * is not a heading.
+   *
+   * Lower it to `'warn'` if a plugin of yours adds ids this package cannot see
+   * at render time.
+   */
+  onBrokenAnchors?: DocsLinkSeverity | undefined;
+  /**
+   * Route prefixes that belong to your application, not to the documentation.
+   *
+   * ⚠️ ONLY MEANINGFUL AT A ROOT MOUNT, WHICH IS ALSO THE ONLY PLACE IT IS
+   * NEEDED. Under `basePath: '/docs'` an absolute link either carries the
+   * prefix — so it is documentation and is checked — or it does not, and this
+   * package leaves it alone. Under `basePath: '/'` there is no prefix to test
+   * against: `/setup` and `/login` look identical, and both are checked against
+   * the published pages.
+   *
+   * That is the right default, because a root mount is what you choose when the
+   * origin serves documentation and nothing else — `docs.example.com` — and
+   * there an unknown absolute link is always a typo. If the origin *does* serve
+   * something else, name what is yours:
+   *
+   * ```ts
+   * externalRoutes: ['/login', '/dashboard', '/api/']
+   * ```
+   *
+   * A link is skipped when it equals one of these or begins with one followed
+   * by `/`. It is a statement about your application, so nothing here can infer
+   * it and nothing tries.
    */
-  assertLinks?: boolean | undefined;
+  externalRoutes?: readonly string[] | undefined;
   /**
    * Validates every page's frontmatter. Defaults to `docFrontmatterSchema`
    * from `@waveso/docs/frontmatter`.
@@ -345,7 +439,9 @@ interface ResolvedDocsConfig<TFrontmatter extends DocFrontmatter = DocFrontmatte
   contentDir: string;
   basePath: string;
   includeDrafts: boolean;
-  assertLinks: boolean;
+  onBrokenLinks: DocsLinkSeverity;
+  onBrokenAnchors: DocsLinkSeverity;
+  externalRoutes: readonly string[];
   /**
    * As supplied. `resolveDocsConfig` omits the key rather than setting it to
    * `undefined` when the built-in `docFrontmatterSchema` applies, so the
@@ -354,4 +450,4 @@ interface ResolvedDocsConfig<TFrontmatter extends DocFrontmatter = DocFrontmatte
   frontmatterSchema?: StandardSchemaV1<unknown, TFrontmatter> | undefined;
 }
 //#endregion
-export { DocFile, DocFrontmatter, DocLinkContext, DocNavGroup, DocNavLink, DocNavNode, DocNavPage, DocNavSeparator, DocsConfig, DocsMeta, ImageResolver, LinkResolver, RenderedDoc, ResolvedDocsConfig, SearchRecord, TocEntry };
+export { DocFile, DocFrontmatter, DocLinkContext, DocNavGroup, DocNavLink, DocNavNode, DocNavPage, DocNavSeparator, DocsConfig, DocsLinkSeverity, DocsMeta, ImageResolver, LinkResolver, RenderedDoc, ResolvedDocsConfig, SearchRecord, TocEntry };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@waveso/docs",
-  "version": "0.4.0",
+  "version": "0.6.0",
   "description": "Zero parser bytes in the browser: markdown docs for Next.js, built to hast in Node and rendered as your components",
   "type": "module",
   "sideEffects": [
@@ -76,6 +76,10 @@
       "types": "./dist/react/markdown-components.d.ts",
       "default": "./dist/react/markdown-components.js"
     },
+    "./react/next-link": {
+      "types": "./dist/react/next-link.d.ts",
+      "default": "./dist/react/next-link.js"
+    },
     "./react/next-search": {
       "types": "./dist/react/next-search.d.ts",
       "default": "./dist/react/next-search.js"
@@ -196,7 +200,6 @@
     "test:smoke": "node smoke/check.ts",
     "size": "node scripts/size.ts",
     "build:site": "next build site",
-    "dev:site": "next dev site",
-    "shoot": "node scripts/shoot.ts"
+    "dev:site": "next dev site"
   }
 }