docspack 0.2.0 → 0.4.0

package/src/build.ts

@@ -11,11 +11,16 @@ import { fetchableLinks, parseLlmsTxt } from "./llms-txt.js";
 import {
   CHUNKS_DIR,
   type ChunkSpec,
+  type DocumentedLibrary,
   estimateTokens,
   LLMS_DIR,
   MANIFEST_FILE,
+  type PackageManifest,
+  parseDocumentedLibrary,
+  parseManifest,
   serializeManifest,
 } from "./spec.js";
+import { STOPWORDS } from "./stopwords.js";
 
 export interface BuildOptions {
   /** Directory the docs package is written to. */
@@ -30,6 +35,14 @@ export interface BuildOptions {
   readonly source?: string;
   readonly pages?: number;
   readonly maxChunkTokens?: number;
+  /**
+   * Merge adjacent sections until a chunk reaches this size, never crossing `maxChunkTokens`.
+   * Off unless set: splitting at every heading is right for prose, and wrong only for generated
+   * reference, where a heading is a field name.
+   */
+  readonly minChunkTokens?: number;
+  /** Libraries this package documents, each `name` or `name@version`. Recorded in the manifest. */
+  readonly documents?: readonly string[];
   readonly http?: HttpClient;
   readonly onProgress?: (message: string) => void;
 }
@@ -53,6 +66,8 @@ interface SourceDocument {
   readonly atomic?: boolean;
   readonly tags?: readonly string[];
   readonly entities?: readonly string[];
+  /** Libraries this document describes, when that is narrower than the package's `documents`. */
+  readonly documents?: readonly string[];
 }
 
 const DEFAULT_MAX_CHUNK_TOKENS = 800;
@@ -76,18 +91,34 @@ export async function buildPackage(input: BuildOptions): Promise<BuildResult> {
   }
 
   const maxTokens = options.maxChunkTokens ?? DEFAULT_MAX_CHUNK_TOKENS;
-  const { chunks, files } = chunkDocuments(documents, maxTokens);
+  const minTokens = options.minChunkTokens;
+  if (minTokens !== undefined && minTokens > maxTokens) {
+    throw new DocspackError(
+      `minChunkTokens (${minTokens}) is larger than maxChunkTokens (${maxTokens})`,
+      { hint: "A chunk cannot be required to be bigger than it is allowed to be." },
+    );
+  }
+  const { chunks, files, collisions } = chunkDocuments(documents, maxTokens, minTokens);
 
   const llmsDir = join(options.out, LLMS_DIR);
+  // Read before the payload is removed: a chunk id is what `feedback add --chunk` pins and what
+  // an answer is headed with, so a rebuild that renames one silently orphans both.
+  warnings.push(...idWarnings(collisions, await departedChunkIds(llmsDir, chunks)));
   await rm(llmsDir, { recursive: true, force: true });
   await mkdir(join(llmsDir, CHUNKS_DIR), { recursive: true });
 
   for (const file of files) {
     await writeFile(join(llmsDir, file.path), file.contents, "utf8");
   }
+  const libraries: DocumentedLibrary[] = (options.documents ?? []).map(parseDocumentedLibrary);
   await writeFile(
     join(llmsDir, MANIFEST_FILE),
-    serializeManifest({ name: identity.name, version: identity.version, chunks }),
+    serializeManifest({
+      name: identity.name,
+      version: identity.version,
+      ...(libraries.length === 0 ? {} : { documents: libraries }),
+      chunks,
+    }),
     "utf8",
   );
   await writeFile(join(options.out, "llms.txt"), renderLlmsTxt(identity.name, chunks), "utf8");
@@ -114,7 +145,7 @@ export async function buildPackage(input: BuildOptions): Promise<BuildResult> {
     version: identity.version,
     dir: options.out,
     chunks: chunks.length,
-    tokens: chunks.reduce((total, chunk) => total + chunk.tokens, 0),
+    tokens: chunks.reduce((total, chunk) => total + (chunk.tokens ?? 0), 0),
     warnings,
   };
 }
@@ -142,6 +173,12 @@ async function withConfig(options: BuildOptions): Promise<BuildOptions> {
     ...(options.maxChunkTokens === undefined && config.maxChunkTokens !== undefined
       ? { maxChunkTokens: config.maxChunkTokens }
       : {}),
+    ...(options.minChunkTokens === undefined && config.minChunkTokens !== undefined
+      ? { minChunkTokens: config.minChunkTokens }
+      : {}),
+    ...(options.documents === undefined && config.documents !== undefined
+      ? { documents: config.documents }
+      : {}),
     ...(options.pages === undefined && config.pages !== undefined ? { pages: config.pages } : {}),
   };
 }
@@ -218,6 +255,7 @@ async function collectFromDirectory(dir: string): Promise<SourceDocument[]> {
       origin: relativePath,
       text: cleaned.text,
       ...(cleaned.tags.length === 0 ? {} : { tags: cleaned.tags }),
+      ...(cleaned.documents.length === 0 ? {} : { documents: cleaned.documents }),
     });
   }
   return documents;
@@ -396,21 +434,31 @@ function renderOperation(
 function chunkDocuments(
   documents: readonly SourceDocument[],
   maxTokens: number,
-): { chunks: ChunkSpec[]; files: { path: string; contents: string }[] } {
+  minTokens: number | undefined,
+): {
+  chunks: ChunkSpec[];
+  files: { path: string; contents: string }[];
+  collisions: string[];
+} {
   const chunks: ChunkSpec[] = [];
   const files: { path: string; contents: string }[] = [];
+  const collisions: string[] = [];
   const taken = new Set<string>();
 
   for (const document of documents) {
+    const text = collapseTablePadding(document.text);
     const sections =
       document.atomic === true
-        ? [{ heading: document.title, body: document.text }]
-        : splitMarkdown(document.text, document.title, maxTokens);
+        ? [{ heading: document.title, body: text, level: 0 }]
+        : splitMarkdown(text, document.title, maxTokens, minTokens);
 
     for (const section of sections) {
       const directives = readDirectives(section.body);
       const body = withoutDuplicateHeading(directives.body, section.heading);
-      const id = uniqueId(chunkSlug(document.title, section.heading), taken);
+      const derived = chunkSlug(document.title, section.heading);
+      const id = uniqueId(derived, taken);
+      if (id !== derived) collisions.push(derived);
+      const libraries = directives.documents.length > 0 ? directives.documents : document.documents;
       const contents = `# ${section.heading}\n\n<!-- docspack: from ${document.origin} -->\n\n${body}\n`;
       const file = `${CHUNKS_DIR}/${id}.md`;
 
@@ -419,11 +467,9 @@ function chunkDocuments(
         id,
         file,
         tokens: estimateTokens(contents),
-        // Directive tags first: they are the author aiming one chunk, and `uniqueWords` caps.
-        tags: uniqueWords([
-          ...directives.tags,
+        tags: chunkTags(directives.tags, [
           document.title,
-          section.heading,
+          ...(section.headings ?? [section.heading]),
           ...(document.tags ?? []),
         ]),
         entities: [
@@ -433,24 +479,90 @@ function chunkDocuments(
             ...extractEntities(body),
           ]),
         ],
+        ...(libraries === undefined || libraries.length === 0
+          ? {}
+          : { documents: libraries.map(parseDocumentedLibrary) }),
       });
     }
   }
 
-  return { chunks, files };
+  return { chunks, files, collisions };
+}
+
+/**
+ * The two ways a chunk id moves without anyone deciding it should. Both are reported as one
+ * line each, however many ids are involved: a large package collides in dozens of places, and a
+ * warning per id is a wall of text nobody reads.
+ */
+function idWarnings(collisions: readonly string[], gone: readonly string[]): string[] {
+  const warnings: string[] = [];
+  if (collisions.length > 0) {
+    warnings.push(
+      `${plural(collisions.length, "derived chunk id")} collided and ${collisions.length === 1 ? "was" : "were"} given a numeric suffix: ${list(collisions)} — which section keeps the bare id depends on the order the sources were read in`,
+    );
+  }
+  if (gone.length > 0) {
+    warnings.push(
+      `${plural(gone.length, "chunk id")} the previous build published ${gone.length === 1 ? "is" : "are"} gone: ${list(gone)} — recorded feedback and published links pinned to ${gone.length === 1 ? "it" : "them"} no longer resolve`,
+    );
+  }
+  return warnings;
+}
+
+const plural = (count: number, noun: string): string => `${count} ${noun}${count === 1 ? "" : "s"}`;
+
+const list = (ids: readonly string[]): string =>
+  `${ids.slice(0, 5).join(", ")}${ids.length > 5 ? `, and ${ids.length - 5} more` : ""}`;
+
+/**
+ * Chunk ids the payload on disk carries that this build does not, or nothing when there is no
+ * previous payload to compare with. An unreadable manifest is not a finding: `build` is what
+ * replaces it.
+ */
+async function departedChunkIds(llmsDir: string, chunks: readonly ChunkSpec[]): Promise<string[]> {
+  let previous: PackageManifest;
+  try {
+    previous = parseManifest(
+      JSON.parse(await readFile(join(llmsDir, MANIFEST_FILE), "utf8")),
+      `${LLMS_DIR}/${MANIFEST_FILE}`,
+    );
+  } catch {
+    return [];
+  }
+
+  const current = new Set(chunks.map((chunk) => chunk.id));
+  return previous.chunks.map((chunk) => chunk.id).filter((id) => !current.has(id));
+}
+
+/** One piece of a document, before it becomes a chunk. */
+interface Section {
+  readonly heading: string;
+  readonly body: string;
+  /** Depth of the heading in the source, or 0 when it was inherited rather than written. */
+  readonly level: number;
+  /**
+   * Headings to index as tags, when they are not just `heading`. A packed chunk answers for every
+   * section in it, so all of their headings are search terms for it.
+   */
+  readonly headings?: readonly string[];
 }
 
 /**
  * Splits Markdown at `##` headings, then `###`, then paragraphs, until every piece fits the token
  * budget. Retrieval works best when a chunk answers one question.
+ *
+ * With `minTokens` set, adjacent pieces are then packed back together up to the budget. Every
+ * API generator emits pages whose headings are field names — `Category` is one line, `Sizes` is
+ * three — and one chunk per heading there is hundreds of chunks too small to answer anything.
  */
 function splitMarkdown(
   text: string,
   title: string,
   maxTokens: number,
-): { heading: string; body: string }[] {
-  const sections = splitAtLevel(text, 2, title);
-  const result: { heading: string; body: string }[] = [];
+  minTokens: number | undefined,
+): Section[] {
+  const sections = splitAtLevel(text, 2, title, 0);
+  const result: Section[] = [];
 
   for (const section of sections) {
     if (estimateTokens(section.body) <= maxTokens) {
@@ -458,7 +570,7 @@ function splitMarkdown(
       continue;
     }
 
-    const deeper = splitAtLevel(section.body, 3, section.heading);
+    const deeper = splitAtLevel(section.body, 3, section.heading, section.level);
     for (const part of deeper) {
       if (estimateTokens(part.body) <= maxTokens) {
         if (part.body.trim().length > 0) result.push(part);
@@ -468,22 +580,26 @@ function splitMarkdown(
         ...splitParagraphs(part.body, maxTokens).map((body, index) => ({
           heading: index === 0 ? part.heading : `${part.heading} (${index + 1})`,
           body,
+          // A continuation carries no heading of its own, so packing must not re-emit one.
+          level: index === 0 ? part.level : 0,
         })),
       );
     }
   }
 
-  return result.length > 0 ? result : [{ heading: title, body: text }];
+  const pieces = result.length > 0 ? result : [{ heading: title, body: text, level: 0 }];
+  return minTokens === undefined ? pieces : packSections(pieces, title, minTokens, maxTokens);
 }
 
 function splitAtLevel(
   text: string,
   level: number,
   fallbackHeading: string,
-): { heading: string; body: string }[] {
+  fallbackLevel: number,
+): Section[] {
   const marker = new RegExp(`^#{${level}}\\s+(.+?)\\s*$`);
-  const sections: { heading: string; body: string[] }[] = [];
-  let current = { heading: fallbackHeading, body: [] as string[] };
+  const sections: { heading: string; body: string[]; level: number }[] = [];
+  let current = { heading: fallbackHeading, body: [] as string[], level: fallbackLevel };
   let inFence = false;
 
   for (const line of text.split("\n")) {
@@ -492,7 +608,7 @@ function splitAtLevel(
     const match = inFence ? null : marker.exec(line);
     if (match?.[1] !== undefined) {
       if (current.body.join("\n").trim().length > 0) sections.push(current);
-      current = { heading: match[1], body: [] };
+      current = { heading: match[1], body: [], level };
       continue;
     }
     current.body.push(line);
@@ -501,10 +617,57 @@ function splitAtLevel(
 
   return sections.map((section) => ({
     heading: section.heading,
+    level: section.level,
     body: section.body.join("\n").trim(),
   }));
 }
 
+/**
+ * Merges adjacent sections until a group reaches `minTokens`, never crossing `maxTokens`. A
+ * merged group is headed by the document title and keeps each section's own heading inside it,
+ * so a whole component lands in one chunk without losing the structure a reader needs.
+ */
+function packSections(
+  sections: readonly Section[],
+  title: string,
+  minTokens: number,
+  maxTokens: number,
+): Section[] {
+  const packed: Section[] = [];
+  let group: Section[] = [];
+  let tokens = 0;
+
+  const flush = (): void => {
+    if (group.length === 1 && group[0] !== undefined) packed.push(group[0]);
+    else if (group.length > 1) {
+      packed.push({
+        heading: title,
+        level: 0,
+        body: group.map(withHeading).join("\n\n"),
+        headings: group.map((section) => section.heading),
+      });
+    }
+    group = [];
+    tokens = 0;
+  };
+
+  for (const section of sections) {
+    const size = estimateTokens(withHeading(section));
+    if (group.length > 0 && (tokens >= minTokens || tokens + size > maxTokens)) flush();
+    group.push(section);
+    tokens += size;
+  }
+  flush();
+
+  return packed;
+}
+
+/** A section as it appeared in the source, heading included when it had one. */
+function withHeading(section: Section): string {
+  if (section.level === 0) return section.body;
+  return `${"#".repeat(section.level)} ${section.heading}\n\n${section.body}`;
+}
+
 function splitParagraphs(text: string, maxTokens: number): string[] {
   const parts: string[] = [];
   let buffer: string[] = [];
@@ -562,26 +725,36 @@ function matchAll(text: string, pattern: RegExp): string[] {
 }
 
 /** One authored override under a heading: `<!-- docspack: tags=grid,columns -->`. */
-const DIRECTIVE = /^[ \t]*<!--[ \t]*docspack:[ \t]*(tags|entities)[ \t]*=([^>]*?)-->[ \t]*$/gim;
+const DIRECTIVE =
+  /^[ \t]*<!--[ \t]*docspack:[ \t]*(tags|entities|documents)[ \t]*=([^>]*?)-->[ \t]*$/gim;
 
 /**
  * Reads per-section directives and removes them from the body. Front matter belongs to a whole
  * document, so a page of eighty sections cannot aim any one of them; this is that lever.
  */
-function readDirectives(body: string): { body: string; tags: string[]; entities: string[] } {
-  const tags: string[] = [];
-  const entities: string[] = [];
+function readDirectives(body: string): {
+  body: string;
+  tags: string[];
+  entities: string[];
+  documents: string[];
+} {
+  const collected: Record<string, string[]> = { tags: [], entities: [], documents: [] };
 
   const text = body.replace(DIRECTIVE, (_line, key: string, value: string) => {
     const values = value
       .split(",")
       .map((item) => item.trim())
       .filter((item) => item.length > 0);
-    (key.toLowerCase() === "tags" ? tags : entities).push(...values);
+    collected[key.toLowerCase()]?.push(...values);
     return "";
   });
 
-  return { body: text.replace(/\n{3,}/g, "\n\n").trim(), tags, entities };
+  return {
+    body: text.replace(/\n{3,}/g, "\n\n").trim(),
+    tags: collected.tags ?? [],
+    entities: collected.entities ?? [],
+    documents: collected.documents ?? [],
+  };
 }
 
 /**
@@ -620,14 +793,69 @@ function slugify(input: string, fallback: string): string {
   return slug.length > 0 ? slug : fallback;
 }
 
-function uniqueWords(inputs: readonly string[]): string[] {
-  const words = new Set<string>();
+const MAX_TAGS = 24;
+
+/**
+ * The words indexed alongside a chunk.
+ *
+ * Directive tags are taken as the author wrote them — those are deliberate, and one of them
+ * being a function word is the author's decision. Words lifted out of a title or a heading are
+ * filtered: nobody chose them as search terms, and the tag field is weighted 3× in the ranking,
+ * which is what made "How to use it" the answer to every question beginning "how do I".
+ */
+function chunkTags(authored: readonly string[], derived: readonly string[]): string[] {
+  const tags = new Set(words(authored));
+  for (const word of words(derived)) {
+    if (!STOPWORDS.has(word)) tags.add(word);
+  }
+  return [...tags].slice(0, MAX_TAGS);
+}
+
+function words(inputs: readonly string[]): string[] {
+  const found: string[] = [];
   for (const input of inputs) {
-    for (const word of input.toLowerCase().match(/[\p{L}\p{N}]{2,}/gu) ?? []) {
-      words.add(word);
-    }
+    found.push(...(input.toLowerCase().match(/[\p{L}\p{N}]{2,}/gu) ?? []));
   }
-  return [...words].slice(0, 24);
+  return found;
+}
+
+/** A row of a Markdown table: `| Name | Type |`, and the `|---|---:|` rule under it. */
+const TABLE_ROW = /^\|.*\|/;
+const TABLE_RULE = /^[|\-: \t]+$/;
+
+/**
+ * Collapses the alignment padding out of Markdown tables.
+ *
+ * Every API generator pads table cells into columns. That padding is whitespace to a reader and
+ * to a renderer, but tokens are estimated from characters, so a generated props table is
+ * measured at several times its real weight — which spends the response budget, trips
+ * `chunk-too-large`, and forces splits that leave fragments competing for the same query. One
+ * chart's props table measured 4,094 tokens padded and 1,245 collapsed.
+ *
+ * Runs inside a code fence are left alone; a fenced table is example text, and its alignment is
+ * the thing being shown.
+ */
+export function collapseTablePadding(text: string): string {
+  let inFence = false;
+
+  return text
+    .split("\n")
+    .map((line) => {
+      if (/^\s*(?:```|~~~)/.test(line)) {
+        inFence = !inFence;
+        return line;
+      }
+      if (inFence) return line;
+
+      const indent = /^[ \t]*/.exec(line)?.[0] ?? "";
+      const rest = line.slice(indent.length);
+      // Four spaces or more is an indented code block, whatever the line looks like.
+      if (indent.length >= 4 || !TABLE_ROW.test(rest)) return line;
+
+      const collapsed = rest.replace(/[ \t]{2,}/g, " ");
+      return indent + (TABLE_RULE.test(rest) ? collapsed.replace(/-{4,}/g, "---") : collapsed);
+    })
+    .join("\n");
 }
 
 function uniqueId(base: string, taken: Set<string>): string {