@waveso/docs 0.1.0 → 0.2.0

package/dist/search-index.js

@@ -1,5 +1,6 @@
-import { SEARCH_INDEX_OPTIONS } from "./search-options.js";
-import { mkdir, writeFile } from "node:fs/promises";
+import { isFootnotes, isTransparentContainer } from "./section-boundary.js";
+import { mergeSearchOptions } from "./search-options.js";
+import { mkdir, rename, rm, writeFile } from "node:fs/promises";
 import path from "node:path";
 import MiniSearch from "minisearch";
 //#region src/search-index.ts
@@ -16,8 +17,6 @@ import MiniSearch from "minisearch";
 * 1.35 ms / 3.84 ms. Fuse is a fuzzy short-string matcher routinely
 * misapplied to full text.
 */
-/** Default excerpt length, in characters, of {@link SearchRecord.text}. */
-const DEFAULT_EXCERPT_LENGTH = 300;
 /** `<h1>`…`<h6>` to their numeric depth. */
 const HEADING_DEPTHS = /* @__PURE__ */ new Map([
 	["h1", 1],
@@ -43,32 +42,6 @@ const SKIPPED_TAGS = /* @__PURE__ */ new Set([
 	"svg",
 	"template"
 ]);
-/**
-* Containers that merely wrap content rather than nesting it semantically.
-*
-* Walking through them keeps heading detection working when a rehype plugin
-* (or a consumer's own) wraps the document body, without descending into
-* blockquotes or list items where a heading is not a section boundary.
-*/
-const TRANSPARENT_TAGS = /* @__PURE__ */ new Set([
-	"div",
-	"section",
-	"article",
-	"main"
-]);
-/**
-* The GFM footnote block `mdast-util-to-hast` appends, and everything in it.
-*
-* It is a `section` — so {@link TRANSPARENT_TAGS} would otherwise walk straight
-* into it — carrying a generated `<h2 id="footnote-label">Footnotes</h2>`. That
-* heading is machinery, not a section of the page: indexing it puts a
-* `Footnotes` hit in the dialog for every footnoted page, and the footnote text
-* itself is already indexed where it was written. `rehypeCaptureToc` skips the
-* same subtree, so the TOC and the index agree.
-*/
-function isFootnotes(node) {
-	return node.properties.dataFootnotes !== void 0;
-}
 /** Tags after which extracted text needs a separator. */
 const BLOCK_TAGS = /* @__PURE__ */ new Set([
 	"address",
@@ -118,13 +91,20 @@ const BLOCK_TAGS = /* @__PURE__ */ new Set([
 * does with the same heading — the two must not disagree about which sections
 * exist.
 *
+* `text` is the section's PROSE IN FULL. It used to be cut to 300 characters
+* before indexing, which on a normal corpus (200 pages × 6 sections, ~1,686
+* characters of prose each) dropped 82% of the words from the index — and
+* `combineWith: 'AND'` compounds it, since every term of a query then has to
+* land inside the surviving prefix of the same section. The cap bought nothing
+* back: `storeFields` does not carry `text`, so not one character of the kept
+* prefix was ever rendered. Truncate for display, in the layer that displays.
+*
 * Not generic over the frontmatter type, deliberately: `frontmatter.title` is
 * the only field read, and a `RenderedDoc` carrying a project's own fields is
 * assignable to this signature already. A type parameter here would appear in
 * every call site and constrain nothing.
 */
-function extractSearchRecords(doc, options = {}) {
-	const excerptLength = options.excerptLength ?? DEFAULT_EXCERPT_LENGTH;
+function extractSearchRecords(doc) {
 	const title = doc.frontmatter.title;
 	const records = [];
 	const slug = doc.segments.join("/");
@@ -145,7 +125,7 @@ function extractSearchRecords(doc, options = {}) {
 			heading: section.heading,
 			ancestors: section.ancestors,
 			href,
-			text: truncateAtWordBoundary(collapseWhitespace(section.parts.join(" ")), excerptLength)
+			text: collapseWhitespace(section.parts.join(" "))
 		});
 	};
 	for (const node of iterateBlocks(doc.hast.children)) {
@@ -191,7 +171,7 @@ function extractSearchRecords(doc, options = {}) {
 function* iterateBlocks(nodes) {
 	for (const node of nodes) {
 		if (node.type === "element" && isFootnotes(node)) continue;
-		if (node.type === "element" && TRANSPARENT_TAGS.has(node.tagName)) yield* iterateBlocks(node.children);
+		if (node.type === "element" && isTransparentContainer(node)) yield* iterateBlocks(node.children);
 		else yield node;
 	}
 }
@@ -235,24 +215,21 @@ function collapseWhitespace(text) {
 	return text.replace(/\s+/g, " ").trim();
 }
 /**
-* Cut `text` to at most `limit` characters, on a word boundary where one is
-* near enough to the cut to be worth honouring.
-*/
-function truncateAtWordBoundary(text, limit) {
-	if (limit <= 0) return "";
-	if (text.length <= limit) return text;
-	const slice = text.slice(0, limit);
-	const lastSpace = slice.lastIndexOf(" ");
-	return `${(lastSpace > limit / 2 ? slice.slice(0, lastSpace) : slice).trimEnd()}…`;
-}
-/**
 * Build a serialised MiniSearch index from extracted records.
 *
 * The return value is JSON, ready for `MiniSearch.loadJSON` on the client or
-* for {@link writeSearchIndex} to put on disk.
+* for {@link writeSearchIndex} to put on disk. The output is byte-stable for a
+* given record list, so an index committed to the repository does not dirty
+* the diff on every build.
+*
+* ⚠️ `options` MUST ALSO REACH THE DIALOG — pass the identical object to
+* `SearchDialog`'s `searchOptions`. Both sides feed it through
+* `mergeSearchOptions`, and a `tokenize` or `processTerm` applied to the
+* documents but not to the query produces an index whose terms no query can
+* spell: zero results, no error, nothing in the console.
 */
-function buildSearchIndex(records) {
-	const index = new MiniSearch(SEARCH_INDEX_OPTIONS);
+function buildSearchIndex(records, options = {}) {
+	const index = new MiniSearch(mergeSearchOptions(options));
 	index.addAll(records);
 	return JSON.stringify(index);
 }
@@ -262,12 +239,29 @@ function buildSearchIndex(records) {
 * Returns the byte size written, so a build step can log it or assert a
 * budget — a docs index that quietly crosses a megabyte is a regression
 * nobody notices until the dialog takes a second to open.
+*
+* ⚠️ WRITTEN BESIDE THE TARGET AND RENAMED OVER IT, NEVER INTO IT. The target
+* is normally `public/search-index.json`, a live static asset: writing in
+* place truncates it to zero and grows it back in 1 MiB chunks, and a fetch
+* landing in that window gets a 200 with a half-written body. `response.ok`
+* passes, the parse throws, and the dialog is stuck in its error state —
+* *"Try reloading the page"* — for every visitor, reloading forever, until
+* someone redeploys content that did not change. `rename` is atomic within a
+* filesystem, so a reader sees either the whole old file or the whole new one;
+* it also makes two concurrent builds safe.
 */
-async function writeSearchIndex(records, outFile) {
-	const json = buildSearchIndex(records);
+async function writeSearchIndex(records, outFile, options = {}) {
+	const json = buildSearchIndex(records, options);
 	const absolute = path.resolve(outFile);
+	const temporary = `${absolute}.tmp-${process.pid}`;
 	await mkdir(path.dirname(absolute), { recursive: true });
-	await writeFile(absolute, json, "utf8");
+	try {
+		await writeFile(temporary, json, "utf8");
+		await rename(temporary, absolute);
+	} catch (error) {
+		await rm(temporary, { force: true });
+		throw error;
+	}
 	return Buffer.byteLength(json, "utf8");
 }
 //#endregion

package/dist/search-options.d.ts

@@ -1,6 +1,16 @@
 import { SearchRecord } from "./types.js";
 import { Options } from "minisearch";
 //#region src/search-options.d.ts
+/**
+ * Split text into index terms.
+ *
+ * ⚠️ THE SAME FUNCTION MUST TOKENISE THE DOCUMENTS AND THE QUERY. It is part of
+ * {@link SEARCH_INDEX_OPTIONS} rather than a separate argument for exactly that
+ * reason — MiniSearch reads `tokenize` both when indexing and when searching,
+ * and a build-time tokeniser the client does not share fails silently, with an
+ * index full of terms no query can ever spell.
+ */
+declare function tokenizeSearchText(text: string): string[];
 /**
  * Fields, stored fields and query defaults for the docs index.
  *
@@ -9,10 +19,31 @@ import { Options } from "minisearch";
  * recover fields the serialised index never stored, and the failure is silent
  * — results arrive with an id and a score and nothing to render.
  *
+ * `title` is stored but NOT indexed, and the asymmetry is the whole point. A
+ * record exists per section, and every one of them carries its page's title, so
+ * indexing it scored all N sections of one page on a single title match: a
+ * query for `configuration` filled seven of the dialog's eight rows with the
+ * same page and pushed every other page's matches off the list entirely. The
+ * page title is still searchable — the lead record's `heading` IS the title, so
+ * it matches exactly once per page, which is what the reader wanted from it.
+ *
  * `combineWith: 'AND'` is not MiniSearch's default and is not optional here.
  * The default OR returned 68–131 hits on real queries where AND returned a
  * usable handful.
  */
 declare const SEARCH_INDEX_OPTIONS: Options<SearchRecord>;
+/**
+ * {@link SEARCH_INDEX_OPTIONS} with `overrides` applied over it.
+ *
+ * The one supported way to customise search, and it exists so both halves can
+ * customise it identically: `buildSearchIndex` takes the same overrides the
+ * dialog does, and an index built with one `tokenize` and queried with another
+ * returns nothing at all.
+ *
+ * `searchOptions` merges one level deep, so `{ searchOptions: { fuzzy: 0 } }`
+ * keeps `combineWith: 'AND'` instead of silently reverting it to MiniSearch's
+ * OR. Nothing below that merges: a `boost` override replaces the whole map.
+ */
+declare function mergeSearchOptions(overrides?: Partial<Options<SearchRecord>>): Options<SearchRecord>;
 //#endregion
-export { SEARCH_INDEX_OPTIONS };
+export { SEARCH_INDEX_OPTIONS, mergeSearchOptions, tokenizeSearchText };

package/dist/search-options.js

@@ -1,5 +1,39 @@
 //#region src/search-options.ts
 /**
+* Everything that is not a letter, a digit or a combining mark.
+*
+* Applied to each segment the word breaker returns, so `wave.config.json`
+* indexes as three terms rather than one. UAX #29 treats a full stop between
+* letters as part of the word, which is right for `e.g.` in prose and wrong
+* for every dotted identifier in a technical document.
+*/
+const NON_WORD = /[^\p{L}\p{N}\p{M}]+/u;
+/**
+* Absent on browsers older than Firefox 125; Node has had it since 16.
+* Without it CJK falls back to whitespace splitting, which is how this package
+* shipped before: Chinese, Japanese and Thai prose has no spaces, so a whole
+* clause became one term and every query returned zero results.
+*/
+const WORD_SEGMENTER = typeof Intl.Segmenter === "function" ? new Intl.Segmenter("en", { granularity: "word" }) : void 0;
+/**
+* Split text into index terms.
+*
+* ⚠️ THE SAME FUNCTION MUST TOKENISE THE DOCUMENTS AND THE QUERY. It is part of
+* {@link SEARCH_INDEX_OPTIONS} rather than a separate argument for exactly that
+* reason — MiniSearch reads `tokenize` both when indexing and when searching,
+* and a build-time tokeniser the client does not share fails silently, with an
+* index full of terms no query can ever spell.
+*/
+function tokenizeSearchText(text) {
+	if (WORD_SEGMENTER === void 0) return text.split(NON_WORD).filter((token) => token !== "");
+	const tokens = [];
+	for (const { segment, isWordLike } of WORD_SEGMENTER.segment(text)) {
+		if (isWordLike !== true) continue;
+		for (const token of segment.split(NON_WORD)) if (token !== "") tokens.push(token);
+	}
+	return tokens;
+}
+/**
 * Fields, stored fields and query defaults for the docs index.
 *
 * `storeFields` MUST be applied at build time. It decides what
@@ -7,13 +41,20 @@
 * recover fields the serialised index never stored, and the failure is silent
 * — results arrive with an id and a score and nothing to render.
 *
+* `title` is stored but NOT indexed, and the asymmetry is the whole point. A
+* record exists per section, and every one of them carries its page's title, so
+* indexing it scored all N sections of one page on a single title match: a
+* query for `configuration` filled seven of the dialog's eight rows with the
+* same page and pushed every other page's matches off the list entirely. The
+* page title is still searchable — the lead record's `heading` IS the title, so
+* it matches exactly once per page, which is what the reader wanted from it.
+*
 * `combineWith: 'AND'` is not MiniSearch's default and is not optional here.
 * The default OR returned 68–131 hits on real queries where AND returned a
 * usable handful.
 */
 const SEARCH_INDEX_OPTIONS = {
 	fields: [
-		"title",
 		"heading",
 		"text",
 		"ancestors"
@@ -24,17 +65,39 @@ const SEARCH_INDEX_OPTIONS = {
 		"ancestors",
 		"href"
 	],
+	tokenize: tokenizeSearchText,
 	searchOptions: {
 		prefix: true,
 		fuzzy: .2,
 		combineWith: "AND",
 		boost: {
-			title: 4,
 			heading: 3,
 			text: 2,
 			ancestors: 1
 		}
 	}
 };
+/**
+* {@link SEARCH_INDEX_OPTIONS} with `overrides` applied over it.
+*
+* The one supported way to customise search, and it exists so both halves can
+* customise it identically: `buildSearchIndex` takes the same overrides the
+* dialog does, and an index built with one `tokenize` and queried with another
+* returns nothing at all.
+*
+* `searchOptions` merges one level deep, so `{ searchOptions: { fuzzy: 0 } }`
+* keeps `combineWith: 'AND'` instead of silently reverting it to MiniSearch's
+* OR. Nothing below that merges: a `boost` override replaces the whole map.
+*/
+function mergeSearchOptions(overrides = {}) {
+	return {
+		...SEARCH_INDEX_OPTIONS,
+		...overrides,
+		searchOptions: {
+			...SEARCH_INDEX_OPTIONS.searchOptions,
+			...overrides.searchOptions
+		}
+	};
+}
 //#endregion
-export { SEARCH_INDEX_OPTIONS };
+export { SEARCH_INDEX_OPTIONS, mergeSearchOptions, tokenizeSearchText };

package/dist/section-boundary.d.ts

@@ -0,0 +1,17 @@
+import { Element } from "hast";
+//#region src/section-boundary.d.ts
+/** Whether a heading nested in `node` still opens a section of its own. */
+declare function isTransparentContainer(node: Element): boolean;
+/**
+ * The GFM footnote block `mdast-util-to-hast` appends, and everything in it.
+ *
+ * It is a `section` — so {@link TRANSPARENT_TAGS} would otherwise walk straight
+ * into it — carrying a generated `<h2 id="footnote-label">Footnotes</h2>`. That
+ * heading is machinery, not a section of the page: indexing it puts a
+ * `Footnotes` hit in the dialog for every footnoted page, and the footnote text
+ * itself is already indexed where it was written. Both readers skip the same
+ * subtree, so the TOC and the index agree.
+ */
+declare function isFootnotes(node: Element): boolean;
+//#endregion
+export { isFootnotes, isTransparentContainer };

package/dist/section-boundary.js

@@ -0,0 +1,43 @@
+//#region src/section-boundary.ts
+/**
+* Containers a heading can sit inside and still open a section.
+*
+* ⚠️ THIS SET IS THE SECTION-BOUNDARY RULE, AND BOTH READERS HAVE TO AGREE ON
+* IT. A heading the TOC captures but the index does not gets a table-of-contents
+* entry with no searchable section behind it, its prose folded into the section
+* above, and a hit that deep-links to the wrong anchor under the wrong
+* breadcrumb. `callout` — what `> [!NOTE]` renders to in this very package — was
+* exactly that case. Use {@link isTransparentContainer} rather than reaching for
+* the set, and add to it in one place.
+*
+* `li` is deliberately absent, and not as an oversight: a list item's children
+* can be bare inline nodes, so descending into one would yield `re`, `<em>al`,
+* `ly` as three separate blocks and index `re al ly` as three terms.
+*/
+const TRANSPARENT_TAGS = /* @__PURE__ */ new Set([
+	"article",
+	"blockquote",
+	"callout",
+	"div",
+	"main",
+	"section"
+]);
+/** Whether a heading nested in `node` still opens a section of its own. */
+function isTransparentContainer(node) {
+	return TRANSPARENT_TAGS.has(node.tagName);
+}
+/**
+* The GFM footnote block `mdast-util-to-hast` appends, and everything in it.
+*
+* It is a `section` — so {@link TRANSPARENT_TAGS} would otherwise walk straight
+* into it — carrying a generated `<h2 id="footnote-label">Footnotes</h2>`. That
+* heading is machinery, not a section of the page: indexing it puts a
+* `Footnotes` hit in the dialog for every footnoted page, and the footnote text
+* itself is already indexed where it was written. Both readers skip the same
+* subtree, so the TOC and the index agree.
+*/
+function isFootnotes(node) {
+	return node.properties.dataFootnotes !== void 0;
+}
+//#endregion
+export { isFootnotes, isTransparentContainer };

package/dist/source.d.ts

@@ -9,6 +9,17 @@ import { DocFile, DocFrontmatter, DocNavNode, DocsConfig, ResolvedDocsConfig } f
 interface DocsSource<TFrontmatter extends DocFrontmatter = DocFrontmatter> {
   /** Every page, drafts excluded unless `includeDrafts`. */
   all(): Promise<Array<DocFile<TFrontmatter>>>;
+  /**
+   * Every `draft: true` page, whatever `includeDrafts` says.
+   *
+   * A link to a draft resolves to a file plainly sitting on disk, so the
+   * renderer's generic "no such page exists" sends the author hunting for a
+   * typo in a link that is spelled correctly. `createDocsRoute` feeds this to
+   * `DocsRendererOptions.draftRoutes` so the build still fails, but names the
+   * reason. Unfiltered on purpose: under `includeDrafts` these routes are
+   * published too, and the renderer checks `knownRoutes` first.
+   */
+  drafts(): Promise<Array<DocFile<TFrontmatter>>>;
   /**
    * One page by route segments. Returns drafts regardless of config, so a
    * preview route can opt into them without a second source.
@@ -52,7 +63,8 @@ declare function createDocsSource<TFrontmatter extends DocFrontmatter = DocFront
  *
  * Shared by both adapters so they agree on which routes exist: an alias is a
  * redirect the host installs, so a link to one resolves, and a link that
- * builds under Next must build under Vite.
+ * builds under Next must build under Vite. The source scan calls it too, so
+ * every rejection below names the markdown file at the moment it is read.
  */
 declare function toAliasRoute(alias: string, basePath: string,
 /**