docspack 0.1.1 → 0.3.0

package/src/build.ts

@@ -11,11 +11,14 @@ import { fetchableLinks, parseLlmsTxt } from "./llms-txt.js";
 import {
   CHUNKS_DIR,
   type ChunkSpec,
+  type DocumentedLibrary,
   estimateTokens,
   LLMS_DIR,
   MANIFEST_FILE,
+  parseDocumentedLibrary,
   serializeManifest,
 } from "./spec.js";
+import { STOPWORDS } from "./stopwords.js";
 
 export interface BuildOptions {
   /** Directory the docs package is written to. */
@@ -30,6 +33,14 @@ export interface BuildOptions {
   readonly source?: string;
   readonly pages?: number;
   readonly maxChunkTokens?: number;
+  /**
+   * Merge adjacent sections until a chunk reaches this size, never crossing `maxChunkTokens`.
+   * Off unless set: splitting at every heading is right for prose, and wrong only for generated
+   * reference, where a heading is a field name.
+   */
+  readonly minChunkTokens?: number;
+  /** Libraries this package documents, each `name` or `name@version`. Recorded in the manifest. */
+  readonly documents?: readonly string[];
   readonly http?: HttpClient;
   readonly onProgress?: (message: string) => void;
 }
@@ -76,7 +87,14 @@ export async function buildPackage(input: BuildOptions): Promise<BuildResult> {
   }
 
   const maxTokens = options.maxChunkTokens ?? DEFAULT_MAX_CHUNK_TOKENS;
-  const { chunks, files } = chunkDocuments(documents, maxTokens);
+  const minTokens = options.minChunkTokens;
+  if (minTokens !== undefined && minTokens > maxTokens) {
+    throw new DocspackError(
+      `minChunkTokens (${minTokens}) is larger than maxChunkTokens (${maxTokens})`,
+      { hint: "A chunk cannot be required to be bigger than it is allowed to be." },
+    );
+  }
+  const { chunks, files } = chunkDocuments(documents, maxTokens, minTokens);
 
   const llmsDir = join(options.out, LLMS_DIR);
   await rm(llmsDir, { recursive: true, force: true });
@@ -85,9 +103,15 @@ export async function buildPackage(input: BuildOptions): Promise<BuildResult> {
   for (const file of files) {
     await writeFile(join(llmsDir, file.path), file.contents, "utf8");
   }
+  const libraries: DocumentedLibrary[] = (options.documents ?? []).map(parseDocumentedLibrary);
   await writeFile(
     join(llmsDir, MANIFEST_FILE),
-    serializeManifest({ name: identity.name, version: identity.version, chunks }),
+    serializeManifest({
+      name: identity.name,
+      version: identity.version,
+      ...(libraries.length === 0 ? {} : { documents: libraries }),
+      chunks,
+    }),
     "utf8",
   );
   await writeFile(join(options.out, "llms.txt"), renderLlmsTxt(identity.name, chunks), "utf8");
@@ -114,7 +138,7 @@ export async function buildPackage(input: BuildOptions): Promise<BuildResult> {
     version: identity.version,
     dir: options.out,
     chunks: chunks.length,
-    tokens: chunks.reduce((total, chunk) => total + chunk.tokens, 0),
+    tokens: chunks.reduce((total, chunk) => total + (chunk.tokens ?? 0), 0),
     warnings,
   };
 }
@@ -142,6 +166,12 @@ async function withConfig(options: BuildOptions): Promise<BuildOptions> {
     ...(options.maxChunkTokens === undefined && config.maxChunkTokens !== undefined
       ? { maxChunkTokens: config.maxChunkTokens }
       : {}),
+    ...(options.minChunkTokens === undefined && config.minChunkTokens !== undefined
+      ? { minChunkTokens: config.minChunkTokens }
+      : {}),
+    ...(options.documents === undefined && config.documents !== undefined
+      ? { documents: config.documents }
+      : {}),
     ...(options.pages === undefined && config.pages !== undefined ? { pages: config.pages } : {}),
   };
 }
@@ -396,20 +426,24 @@ function renderOperation(
 function chunkDocuments(
   documents: readonly SourceDocument[],
   maxTokens: number,
+  minTokens: number | undefined,
 ): { chunks: ChunkSpec[]; files: { path: string; contents: string }[] } {
   const chunks: ChunkSpec[] = [];
   const files: { path: string; contents: string }[] = [];
   const taken = new Set<string>();
 
   for (const document of documents) {
+    const text = collapseTablePadding(document.text);
     const sections =
       document.atomic === true
-        ? [{ heading: document.title, body: document.text }]
-        : splitMarkdown(document.text, document.title, maxTokens);
+        ? [{ heading: document.title, body: text, level: 0 }]
+        : splitMarkdown(text, document.title, maxTokens, minTokens);
 
     for (const section of sections) {
-      const id = uniqueId(slugify(`${document.title}-${section.heading}`, "chunk"), taken);
-      const contents = `# ${section.heading}\n\n<!-- docspack: from ${document.origin} -->\n\n${section.body}\n`;
+      const directives = readDirectives(section.body);
+      const body = withoutDuplicateHeading(directives.body, section.heading);
+      const id = uniqueId(chunkSlug(document.title, section.heading), taken);
+      const contents = `# ${section.heading}\n\n<!-- docspack: from ${document.origin} -->\n\n${body}\n`;
       const file = `${CHUNKS_DIR}/${id}.md`;
 
       files.push({ path: file, contents });
@@ -417,8 +451,18 @@ function chunkDocuments(
         id,
         file,
         tokens: estimateTokens(contents),
-        tags: uniqueWords([document.title, section.heading, ...(document.tags ?? [])]),
-        entities: [...new Set([...(document.entities ?? []), ...extractEntities(section.body)])],
+        tags: chunkTags(directives.tags, [
+          document.title,
+          ...(section.headings ?? [section.heading]),
+          ...(document.tags ?? []),
+        ]),
+        entities: [
+          ...new Set([
+            ...(document.entities ?? []),
+            ...directives.entities,
+            ...extractEntities(body),
+          ]),
+        ],
       });
     }
   }
@@ -426,17 +470,35 @@ function chunkDocuments(
   return { chunks, files };
 }
 
+/** One piece of a document, before it becomes a chunk. */
+interface Section {
+  readonly heading: string;
+  readonly body: string;
+  /** Depth of the heading in the source, or 0 when it was inherited rather than written. */
+  readonly level: number;
+  /**
+   * Headings to index as tags, when they are not just `heading`. A packed chunk answers for every
+   * section in it, so all of their headings are search terms for it.
+   */
+  readonly headings?: readonly string[];
+}
+
 /**
  * Splits Markdown at `##` headings, then `###`, then paragraphs, until every piece fits the token
  * budget. Retrieval works best when a chunk answers one question.
+ *
+ * With `minTokens` set, adjacent pieces are then packed back together up to the budget. Every
+ * API generator emits pages whose headings are field names — `Category` is one line, `Sizes` is
+ * three — and one chunk per heading there is hundreds of chunks too small to answer anything.
  */
 function splitMarkdown(
   text: string,
   title: string,
   maxTokens: number,
-): { heading: string; body: string }[] {
-  const sections = splitAtLevel(text, 2, title);
-  const result: { heading: string; body: string }[] = [];
+  minTokens: number | undefined,
+): Section[] {
+  const sections = splitAtLevel(text, 2, title, 0);
+  const result: Section[] = [];
 
   for (const section of sections) {
     if (estimateTokens(section.body) <= maxTokens) {
@@ -444,7 +506,7 @@ function splitMarkdown(
       continue;
     }
 
-    const deeper = splitAtLevel(section.body, 3, section.heading);
+    const deeper = splitAtLevel(section.body, 3, section.heading, section.level);
     for (const part of deeper) {
       if (estimateTokens(part.body) <= maxTokens) {
         if (part.body.trim().length > 0) result.push(part);
@@ -454,22 +516,26 @@ function splitMarkdown(
         ...splitParagraphs(part.body, maxTokens).map((body, index) => ({
           heading: index === 0 ? part.heading : `${part.heading} (${index + 1})`,
           body,
+          // A continuation carries no heading of its own, so packing must not re-emit one.
+          level: index === 0 ? part.level : 0,
         })),
       );
     }
   }
 
-  return result.length > 0 ? result : [{ heading: title, body: text }];
+  const pieces = result.length > 0 ? result : [{ heading: title, body: text, level: 0 }];
+  return minTokens === undefined ? pieces : packSections(pieces, title, minTokens, maxTokens);
 }
 
 function splitAtLevel(
   text: string,
   level: number,
   fallbackHeading: string,
-): { heading: string; body: string }[] {
+  fallbackLevel: number,
+): Section[] {
   const marker = new RegExp(`^#{${level}}\\s+(.+?)\\s*$`);
-  const sections: { heading: string; body: string[] }[] = [];
-  let current = { heading: fallbackHeading, body: [] as string[] };
+  const sections: { heading: string; body: string[]; level: number }[] = [];
+  let current = { heading: fallbackHeading, body: [] as string[], level: fallbackLevel };
   let inFence = false;
 
   for (const line of text.split("\n")) {
@@ -478,7 +544,7 @@ function splitAtLevel(
     const match = inFence ? null : marker.exec(line);
     if (match?.[1] !== undefined) {
       if (current.body.join("\n").trim().length > 0) sections.push(current);
-      current = { heading: match[1], body: [] };
+      current = { heading: match[1], body: [], level };
       continue;
     }
     current.body.push(line);
@@ -487,10 +553,57 @@ function splitAtLevel(
 
   return sections.map((section) => ({
     heading: section.heading,
+    level: section.level,
     body: section.body.join("\n").trim(),
   }));
 }
 
+/**
+ * Merges adjacent sections until a group reaches `minTokens`, never crossing `maxTokens`. A
+ * merged group is headed by the document title and keeps each section's own heading inside it,
+ * so a whole component lands in one chunk without losing the structure a reader needs.
+ */
+function packSections(
+  sections: readonly Section[],
+  title: string,
+  minTokens: number,
+  maxTokens: number,
+): Section[] {
+  const packed: Section[] = [];
+  let group: Section[] = [];
+  let tokens = 0;
+
+  const flush = (): void => {
+    if (group.length === 1 && group[0] !== undefined) packed.push(group[0]);
+    else if (group.length > 1) {
+      packed.push({
+        heading: title,
+        level: 0,
+        body: group.map(withHeading).join("\n\n"),
+        headings: group.map((section) => section.heading),
+      });
+    }
+    group = [];
+    tokens = 0;
+  };
+
+  for (const section of sections) {
+    const size = estimateTokens(withHeading(section));
+    if (group.length > 0 && (tokens >= minTokens || tokens + size > maxTokens)) flush();
+    group.push(section);
+    tokens += size;
+  }
+  flush();
+
+  return packed;
+}
+
+/** A section as it appeared in the source, heading included when it had one. */
+function withHeading(section: Section): string {
+  if (section.level === 0) return section.body;
+  return `${"#".repeat(section.level)} ${section.heading}\n\n${section.body}`;
+}
+
 function splitParagraphs(text: string, maxTokens: number): string[] {
   const parts: string[] = [];
   let buffer: string[] = [];
@@ -508,13 +621,90 @@ function splitParagraphs(text: string, maxTokens: number): string[] {
   return parts;
 }
 
-/** Pulls dotted or called identifiers out of inline code, e.g. `Stripe.setApiKey`. */
+/** A member access in inline code, e.g. `Stripe.setApiKey`. */
+const DOTTED_ENTITY = /`([A-Za-z_$][\w$]*(?:\.[A-Za-z_$][\w$]*)+)\(?\)?`/g;
+/**
+ * An inline-code token that is an identifier on its own: `TwoColumn`, `useSlide`,
+ * `text-accent`. Component libraries name most of their surface without a dot, so requiring
+ * one made `verify` blind to them. Two words are required — a single capitalised word in a
+ * code span is as often a value as an API name.
+ */
+const BARE_ENTITY =
+  /`([A-Za-z][a-z0-9]*(?:[A-Z][A-Za-z0-9]*)+|[a-z][a-z0-9]*(?:-[a-z0-9]+)+)\(?\)?`/g;
+
+const MAX_ENTITIES = 20;
+
+/**
+ * Pulls the identifiers a section names out of its inline code.
+ *
+ * Member accesses and camel or Pascal names come first. A reference page often enumerates
+ * dozens of hyphenated variants — `token-0`, `token-1` — and letting those fill the cap would
+ * drop the component the page is actually about.
+ */
 function extractEntities(body: string): string[] {
-  const found = new Set<string>();
-  for (const match of body.matchAll(/`([A-Za-z_$][\w$]*(?:\.[A-Za-z_$][\w$]*)+)\(?\)?`/g)) {
-    if (match[1] !== undefined) found.add(match[1]);
+  const dotted = matchAll(body, DOTTED_ENTITY);
+  const bare = matchAll(body, BARE_ENTITY);
+  const ordered = [
+    ...dotted,
+    ...bare.filter((name) => !name.includes("-")),
+    ...bare.filter((name) => name.includes("-")),
+  ];
+  return [...new Set(ordered)].slice(0, MAX_ENTITIES);
+}
+
+function matchAll(text: string, pattern: RegExp): string[] {
+  const found: string[] = [];
+  for (const match of text.matchAll(pattern)) {
+    if (match[1] !== undefined) found.push(match[1]);
   }
-  return [...found].slice(0, 20);
+  return found;
+}
+
+/** One authored override under a heading: `<!-- docspack: tags=grid,columns -->`. */
+const DIRECTIVE = /^[ \t]*<!--[ \t]*docspack:[ \t]*(tags|entities)[ \t]*=([^>]*?)-->[ \t]*$/gim;
+
+/**
+ * Reads per-section directives and removes them from the body. Front matter belongs to a whole
+ * document, so a page of eighty sections cannot aim any one of them; this is that lever.
+ */
+function readDirectives(body: string): { body: string; tags: string[]; entities: string[] } {
+  const tags: string[] = [];
+  const entities: string[] = [];
+
+  const text = body.replace(DIRECTIVE, (_line, key: string, value: string) => {
+    const values = value
+      .split(",")
+      .map((item) => item.trim())
+      .filter((item) => item.length > 0);
+    (key.toLowerCase() === "tags" ? tags : entities).push(...values);
+    return "";
+  });
+
+  return { body: text.replace(/\n{3,}/g, "\n\n").trim(), tags, entities };
+}
+
+/**
+ * Drops a source `# Title` that repeats the heading being written above it. A single-`H1`
+ * document has no `##` to split at, so its heading falls back to the title and the body would
+ * otherwise open with the same line twice.
+ */
+function withoutDuplicateHeading(body: string, heading: string): string {
+  const [first = "", ...rest] = body.split("\n");
+  const match = /^#\s+(.+?)\s*$/.exec(first.trim());
+  if (match?.[1] === undefined) return body;
+  if (match[1].trim().toLowerCase() !== heading.trim().toLowerCase()) return body;
+  return rest.join("\n").trim();
+}
+
+/**
+ * The chunk id for a section. When the section heading is the document title — every
+ * single-`H1` page — concatenating the two stutters: `colours-colours`.
+ */
+function chunkSlug(title: string, heading: string): string {
+  const titleSlug = slugify(title, "chunk");
+  return slugify(heading, "chunk") === titleSlug
+    ? titleSlug
+    : slugify(`${title}-${heading}`, "chunk");
 }
 
 /** Reduces a heading to a chunk id: lowercase, hyphen-separated, no path separators. */
@@ -529,14 +719,69 @@ function slugify(input: string, fallback: string): string {
   return slug.length > 0 ? slug : fallback;
 }
 
-function uniqueWords(inputs: readonly string[]): string[] {
-  const words = new Set<string>();
+const MAX_TAGS = 24;
+
+/**
+ * The words indexed alongside a chunk.
+ *
+ * Directive tags are taken as the author wrote them — those are deliberate, and one of them
+ * being a function word is the author's decision. Words lifted out of a title or a heading are
+ * filtered: nobody chose them as search terms, and the tag field is weighted 3× in the ranking,
+ * which is what made "How to use it" the answer to every question beginning "how do I".
+ */
+function chunkTags(authored: readonly string[], derived: readonly string[]): string[] {
+  const tags = new Set(words(authored));
+  for (const word of words(derived)) {
+    if (!STOPWORDS.has(word)) tags.add(word);
+  }
+  return [...tags].slice(0, MAX_TAGS);
+}
+
+function words(inputs: readonly string[]): string[] {
+  const found: string[] = [];
   for (const input of inputs) {
-    for (const word of input.toLowerCase().match(/[\p{L}\p{N}]{2,}/gu) ?? []) {
-      words.add(word);
-    }
+    found.push(...(input.toLowerCase().match(/[\p{L}\p{N}]{2,}/gu) ?? []));
   }
-  return [...words].slice(0, 24);
+  return found;
+}
+
+/** A row of a Markdown table: `| Name | Type |`, and the `|---|---:|` rule under it. */
+const TABLE_ROW = /^\|.*\|/;
+const TABLE_RULE = /^[|\-: \t]+$/;
+
+/**
+ * Collapses the alignment padding out of Markdown tables.
+ *
+ * Every API generator pads table cells into columns. That padding is whitespace to a reader and
+ * to a renderer, but tokens are estimated from characters, so a generated props table is
+ * measured at several times its real weight — which spends the response budget, trips
+ * `chunk-too-large`, and forces splits that leave fragments competing for the same query. One
+ * chart's props table measured 4,094 tokens padded and 1,245 collapsed.
+ *
+ * Runs inside a code fence are left alone; a fenced table is example text, and its alignment is
+ * the thing being shown.
+ */
+export function collapseTablePadding(text: string): string {
+  let inFence = false;
+
+  return text
+    .split("\n")
+    .map((line) => {
+      if (/^\s*(?:```|~~~)/.test(line)) {
+        inFence = !inFence;
+        return line;
+      }
+      if (inFence) return line;
+
+      const indent = /^[ \t]*/.exec(line)?.[0] ?? "";
+      const rest = line.slice(indent.length);
+      // Four spaces or more is an indented code block, whatever the line looks like.
+      if (indent.length >= 4 || !TABLE_ROW.test(rest)) return line;
+
+      const collapsed = rest.replace(/[ \t]{2,}/g, " ");
+      return indent + (TABLE_RULE.test(rest) ? collapsed.replace(/-{4,}/g, "---") : collapsed);
+    })
+    .join("\n");
 }
 
 function uniqueId(base: string, taken: Set<string>): string {