@writedocs/generator 0.7.3 → 0.7.4

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@writedocs/generator",
-  "version": "0.7.3",
+  "version": "0.7.4",
   "description": "Static site generator for docs — a writedocs.json + MDX folder in, a static site out.",
   "type": "module",
   "bin": {

package/src/lib/agent-markdown.js

@@ -0,0 +1,137 @@
+// The Markdown an AI agent reads for a page - llms-full.txt and the per-page
+// .md route. It starts from the page's raw MDX, so everything the build does
+// to that source on the way to HTML has to be redone here, or the agent reads
+// something other than the page: `<Visibility>` blocks, imported .mdx
+// snippets (inlined, as the page shows them) and writedocs.json `variables`.
+//
+// Code is left exactly as written - fenced blocks and inline `code` - the same
+// way the build never substitutes inside code: a page documenting the
+// `[[key]]` syntax, or showing `<Snippet />` in an example, must keep it.
+// Plain JavaScript, so it can be tested with plain Node.
+import fs from 'node:fs';
+import path from 'node:path';
+import matter from 'gray-matter';
+import { applyVisibilityForAgents } from './visibility.js';
+
+/** `text` with `transform` applied to everything outside fenced code blocks
+ *  and inline code spans; code comes back untouched. */
+export function mapOutsideCode(text, transform) {
+  const out = [];
+  let prose = [];
+  let fence = null; // the opening marker (``` or ~~~, maybe longer) while inside a block
+  const flush = () => {
+    if (prose.length) out.push(mapOutsideInlineCode(prose.join('\n'), transform));
+    prose = [];
+  };
+  for (const line of text.split('\n')) {
+    const marker = /^\s*(`{3,}|~{3,})/.exec(line)?.[1];
+    if (fence) {
+      out.push(line);
+      if (marker && marker[0] === fence[0] && marker.length >= fence.length && line.trim() === marker) fence = null;
+    } else if (marker) {
+      flush();
+      fence = marker;
+      out.push(line);
+    } else {
+      prose.push(line);
+    }
+  }
+  flush();
+  return out.join('\n');
+}
+
+function mapOutsideInlineCode(text, transform) {
+  // Backtick runs of equal length delimit a code span (CommonMark).
+  return text
+    .split(/(`+)([\s\S]*?)(\1)(?!`)/)
+    .reduce((acc, part, i, parts) => {
+      const k = i % 4;
+      if (k === 0) acc.push(transform(part));
+      else if (k === 1) acc.push(part + parts[i + 1] + parts[i + 2]);
+      return acc;
+    }, [])
+    .join('');
+}
+
+const PLACEHOLDER = /\[\[\s*([\w.-]+)\s*\]\]/g;
+// A standalone `{key}` expression - not an attribute value (`prop={key}`),
+// which the build leaves alone too.
+const MINTLIFY_PLACEHOLDER = /(?<!=\s*)\{\s*([A-Za-z_$][\w$]*)\s*\}/g;
+
+/** writedocs.json `variables` - `[[key]]`, and Mintlify's `{key}` - replaced
+ *  the way the build replaces them (lib/mdx-substitute-variables.js): outside
+ *  code, and only for keys that exist. */
+export function substituteVariables(text, variables) {
+  if (!variables || Object.keys(variables).length === 0) return text;
+  const has = (key) => Object.prototype.hasOwnProperty.call(variables, key);
+  return mapOutsideCode(text, (prose) =>
+    prose
+      .replace(PLACEHOLDER, (match, key) => (has(key) ? String(variables[key]) : match))
+      .replace(MINTLIFY_PLACEHOLDER, (match, key) => (has(key) ? String(variables[key]) : match))
+  );
+}
+
+const IMPORT_LINE = /^[ \t]*import\s+([A-Za-z_$][\w$]*)\s+from\s+['"]([^'"]+\.mdx?)['"];?[ \t]*$/gm;
+
+function resolveSnippet(spec, fromFile, contentDir) {
+  if (spec.startsWith('/snippets/')) return path.join(contentDir, 'snippets', spec.slice('/snippets/'.length));
+  if (spec.startsWith('./') || spec.startsWith('../')) return path.resolve(path.dirname(fromFile), spec);
+  if (spec.startsWith('/')) return path.join(contentDir, spec.slice(1));
+  return null;
+}
+
+function attributesOf(source) {
+  const props = {};
+  for (const m of source.matchAll(/([A-Za-z_$][\w$-]*)\s*=\s*(?:"([^"]*)"|'([^']*)'|\{\s*["']([^"']*)["']\s*\})/g)) {
+    props[m[1]] = m[2] ?? m[3] ?? m[4];
+  }
+  return props;
+}
+
+function fillProps(snippet, props, children) {
+  return snippet
+    .replace(/\{\s*props\.children\s*\}/g, children ?? '')
+    .replace(/\{\s*props\.([A-Za-z_$][\w$]*)\s*\}/g, (match, key) => (key in props ? props[key] : match));
+}
+
+/** Imported .mdx snippets inlined where the page uses them - the snippet's
+ *  own text, with `{props.x}` filled from the tag's string attributes, and
+ *  its own imports inlined in turn. The import line goes too. A snippet that
+ *  can't be read, or a .jsx/.tsx component, stays as written. */
+export function inlineSnippets(text, { file, contentDir, depth = 0 }) {
+  if (depth > 5) return text;
+  const snippets = new Map();
+  let result = mapOutsideCode(text, (prose) =>
+    prose.replace(IMPORT_LINE, (line, name, spec) => {
+      const target = resolveSnippet(spec, file, contentDir);
+      let raw;
+      try {
+        raw = target && fs.readFileSync(target, 'utf-8');
+      } catch {
+        raw = null;
+      }
+      if (!raw) return line;
+      const body = inlineSnippets(matter(raw).content.trim(), { file: target, contentDir, depth: depth + 1 });
+      snippets.set(name, body);
+      return '';
+    })
+  );
+  for (const [name, body] of snippets) {
+    result = mapOutsideCode(result, (prose) =>
+      prose
+        .replace(new RegExp(`<${name}\\b([^>]*?)\\/>`, 'g'), (_, attrs) => fillProps(body, attributesOf(attrs)))
+        .replace(new RegExp(`<${name}\\b([^>]*)>([\\s\\S]*?)<\\/${name}>`, 'g'), (_, attrs, children) =>
+          fillProps(body, attributesOf(attrs), children.trim())
+        )
+    );
+  }
+  return result.replace(/\n{3,}/g, '\n\n');
+}
+
+/** A page's body as an agent should read it. `file` is the page's absolute
+ *  path (to resolve relative snippet imports). */
+export function markdownForAgents(body, { file, contentDir, variables }) {
+  let text = applyVisibilityForAgents(body ?? '');
+  if (file) text = inlineSnippets(text, { file, contentDir });
+  return substituteVariables(text, variables);
+}

package/src/lib/config-file.js

@@ -4,7 +4,7 @@
 //
 // Windows editors - Notepad, PowerShell 5.1's `Out-File`/`Set-Content
 // -Encoding utf8` - save UTF-8 with a byte order mark. JSON.parse rejects
-// it ("Unexpected token ''"), and the error then blames commas and
+// it ("Unexpected token U+FEFF"), and the error then blames commas and
 // brackets the file doesn't have. Every reader strips it here instead.
 import fs from 'node:fs';
 import path from 'node:path';

package/src/lib/llms-index.ts

@@ -0,0 +1,88 @@
+// llms.txt and its child files (/llms/*.md) - the pages an AI tool can
+// discover, in the site's navigation structure. Shared by llms.txt.ts (the
+// root file) and llms/[...path].md.ts (the child files a big site's llms.txt
+// links to), so both render from the same tree in the same run. The
+// structure and the splitting live in lib/llms.js.
+import { getCollection, type CollectionEntry } from 'astro:content';
+import fs from 'node:fs';
+import path from 'node:path';
+import { loadDocsConfig, normalizeEntryId, findAllPages, resolveSiteUrl, fileIdForEntry } from './config';
+import { buildLlmsTree, renderLlmsFiles } from './llms.js';
+import { navigationPageOrder } from './pages.js';
+import { writedocsTempDir } from './writedocs-temp-dir.js';
+
+type DocsEntry = CollectionEntry<'pages'> | CollectionEntry<'generatedDocs'>;
+
+// Mirrors Mintlify's own llms.txt behavior: a page's frontmatter
+// `description` is cut at the first line break (a multi-paragraph
+// description would blow out a one-line list entry) and at 300 characters,
+// so every entry stays scannable.
+const DESCRIPTION_MAX_CHARS = 300;
+function truncateDescription(description: string | undefined): string | undefined {
+  if (!description) return undefined;
+  const firstLine = description.split('\n')[0].trim();
+  if (!firstLine) return undefined;
+  return firstLine.length > DESCRIPTION_MAX_CHARS ? firstLine.slice(0, DESCRIPTION_MAX_CHARS).trimEnd() + '…' : firstLine;
+}
+
+// Most characters one file holds - llms.txt or a child file. The llms.txt
+// spec sets no limit ("small enough to fit in context"); this is Mintlify's,
+// about 25,000 tokens. Past it, sections move into child files instead of
+// pages being dropped.
+export const LLMS_MAX_CHARS = 100_000;
+
+/** The generated files as Map<path, text> - 'llms.txt', then 'llms/<...>.md'
+ *  when the site needs them - or null when the project has its own
+ *  llms.txt, which replaces all of it. */
+export async function llmsIndexFiles(): Promise<Map<string, string> | null> {
+  const contentDir = process.env.WRITEDOCS_CONTENT_DIR || process.cwd();
+  const packageRoot = process.env.WRITEDOCS_PACKAGE_ROOT || process.cwd();
+  // A hand-authored llms.txt at the project root (next to writedocs.json)
+  // wins outright - same override convention Mintlify documents.
+  if (fs.existsSync(path.join(contentDir, 'llms.txt'))) return null;
+
+  const config = loadDocsConfig(contentDir);
+  const siteUrl = resolveSiteUrl(config); // absolute origin if `domain` is set, else null
+
+  const hasGeneratedDocs = fs.existsSync(path.join(writedocsTempDir(contentDir), 'generated-docs'));
+  const hasPages = findAllPages(contentDir).length > 0;
+  const [pagesEntries, generatedDocsEntries] = await Promise.all([
+    hasPages ? getCollection('pages') : Promise.resolve([]),
+    hasGeneratedDocs ? getCollection('generatedDocs') : Promise.resolve([]),
+  ]);
+  const entries: DocsEntry[] = [...pagesEntries, ...generatedDocsEntries];
+
+  const pages = new Map<string, { title: string; href: string; description?: string; slug: string }>();
+  for (const entry of entries) {
+    // A page that opted out of search-engine indexing (`seo.noindex`, also
+    // left out of sitemap.xml) is left out here too, and a frontmatter `url`
+    // page (Mintlify's external link) has no content of its own.
+    if (entry.data.seo?.noindex || entry.data.url) continue;
+    const slug = normalizeEntryId(entry.id);
+    // The .md route when it exists ([...slug].md.ts - with `contextMenu`,
+    // and never for an OpenAPI page, which renders from the spec), else the
+    // HTML page.
+    const hasMarkdownRoute = config.contextMenu && !entry.data.openapi;
+    const href = (siteUrl ?? '') + (hasMarkdownRoute ? `/${slug}.md` : slug === 'index' ? '/' : `/${slug}/`);
+    let description = truncateDescription(entry.data.description);
+    // Mirrors Mintlify: an OpenAPI operation page's description gets its
+    // "METHOD /path" appended - the page itself renders from the spec.
+    if (entry.data.openapi) description = description ? `${description} (${entry.data.openapi})` : entry.data.openapi;
+    pages.set(fileIdForEntry(contentDir, packageRoot, entry), { title: entry.data.title, href, description, slug });
+  }
+
+  const listed = new Set(navigationPageOrder(config.navigation));
+  const unlisted = [...pages.entries()]
+    .filter(([id]) => !listed.has(id))
+    .sort((a, b) => a[1].slug.localeCompare(b[1].slug))
+    .map(([id]) => id);
+  const tree = buildLlmsTree(config.navigation, (id: string) => pages.get(id) ?? null, unlisted);
+
+  return renderLlmsFiles({
+    name: config.name,
+    description: config.description,
+    tree,
+    maxChars: LLMS_MAX_CHARS,
+    urlFor: (file: string) => `${siteUrl ?? ''}/${file}`,
+  });
+}

package/src/lib/llms.js

@@ -0,0 +1,200 @@
+// Shared by the llms routes (lib/llms-index.ts, llms-full.txt.ts): the order
+// pages appear in, and llms.txt's structure. Plain JavaScript, so it can be
+// tested with plain Node.
+import { navigationPageOrder } from './pages.js';
+
+/** `items` ({ fileId, slug }) in navigation order - the order a reader meets
+ *  the pages in the sidebar - then every page the navigation doesn't list,
+ *  by URL. */
+export function orderByNavigation(items, navigation) {
+  const rank = new Map(navigationPageOrder(navigation).map((id, i) => [id, i]));
+  const at = (item) => rank.get(item.fileId) ?? Infinity;
+  return [...items].sort((a, b) => at(a) - at(b) || a.slug.localeCompare(b.slug));
+}
+
+// --- llms.txt as a tree -------------------------------------------------
+//
+// llms.txt lists every page under headings that follow the navigation -
+// products, versions, languages, tabs, dropdowns, groups. A site too big for
+// one file (MAX_CHARS) keeps llms.txt as a directory: its largest sections
+// move into child files under /llms/, linked from where they were, and a
+// child too big is split the same way. No page is ever left out - the scheme
+// Mintlify moved to from cutting the list off
+// (https://www.mintlify.com/blog/scaling-llms-txt).
+
+const CONTAINERS = [
+  ['tabs', 'tab'],
+  ['versions', 'version'],
+  ['languages', 'language'],
+  ['dropdowns', 'dropdown'],
+  ['products', 'product'],
+];
+
+/** A Section: { label, items: [page], sections: [Section] }. `pageFor(id)`
+ *  gives a listed page ({ title, href, description }) or null for one that
+ *  isn't listed (noindex, an external `url`, missing). A page shows up once,
+ *  where the navigation first lists it; `unlisted` pages (not in the
+ *  navigation at all) go in a trailing "Other pages" section. */
+export function buildLlmsTree(navigation, pageFor, unlisted = []) {
+  const seen = new Set();
+  const page = (id) => {
+    if (seen.has(id)) return null;
+    const found = pageFor(id);
+    if (!found) return null;
+    seen.add(id);
+    return found;
+  };
+  const prune = (section) => section.items.length > 0 || section.sections.length > 0;
+
+  function fromList(list) {
+    const node = { items: [], sections: [] };
+    for (const item of list ?? []) {
+      if (typeof item === 'string') {
+        const p = page(item);
+        if (p) node.items.push(p);
+      } else if (item && typeof item === 'object' && typeof item.group === 'string' && Array.isArray(item.pages)) {
+        const own = typeof item.page === 'string' ? page(item.page) : null;
+        const inner = fromList(item.pages);
+        const section = { label: item.group, items: own ? [own, ...inner.items] : inner.items, sections: inner.sections };
+        if (prune(section)) node.sections.push(section);
+      }
+      // A link leaf ({ href }) isn't a page of this site.
+    }
+    return node;
+  }
+
+  function fromContainer(container) {
+    if (Array.isArray(container)) return fromList(container);
+    if (!container || typeof container !== 'object') return { items: [], sections: [] };
+    if (Array.isArray(container.pages)) return fromList(container.pages);
+    for (const [key, labelKey] of CONTAINERS) {
+      if (!Array.isArray(container[key])) continue;
+      const sections = container[key]
+        .filter((child) => child && typeof child === 'object' && !('href' in child))
+        .map((child) => {
+          const label = labelKey === 'version' || labelKey === 'language' ? child.label ?? child[labelKey] : child[labelKey];
+          return { label: String(label), ...fromContainer(child) };
+        })
+        .filter(prune);
+      return { items: [], sections };
+    }
+    return { items: [], sections: [] };
+  }
+
+  const root = fromContainer(navigation);
+  for (const dropdown of (!Array.isArray(navigation) && navigation?.global?.dropdowns) || []) {
+    if ('href' in dropdown) continue;
+    const section = { label: String(dropdown.dropdown), ...fromContainer(dropdown) };
+    if (prune(section)) root.sections.push(section);
+  }
+  const rest = unlisted.map((id) => page(id)).filter(Boolean);
+  if (rest.length) root.sections.push({ label: 'Other pages', items: rest, sections: [] });
+  return root;
+}
+
+const pageLine = (p) => `- [${p.title}](${p.href})${p.description ? `: ${p.description}` : ''}`;
+const countPages = (s) => s.items.length + s.sections.reduce((n, c) => n + countPages(c), 0);
+const heading = (depth, label) => `${'#'.repeat(Math.min(depth, 6))} ${label}`;
+
+function slugify(label) {
+  return String(label).toLowerCase().replace(/[^a-z0-9]+/g, '-').replace(/^-+|-+$/g, '') || 'section';
+}
+
+/** The body of `node` as Markdown lines: its pages, then each sub-section -
+ *  under its own heading, or, when it's in `split`, as a link to its file. */
+function renderBody(node, depth, split) {
+  const lines = [];
+  if (node.items.length) lines.push(...node.items.map(pageLine), '');
+  for (const section of node.sections) {
+    lines.push(heading(depth, section.label), '');
+    const file = split.get(section);
+    if (file) {
+      const n = countPages(section);
+      lines.push(`- [${section.label}](${file.url}): ${n} page${n === 1 ? '' : 's'}, listed in their own file`, '');
+    } else {
+      lines.push(...renderBody(section, depth + 1, split));
+    }
+  }
+  return lines;
+}
+
+const render = (header, node, depth, split) => [...header, ...renderBody(node, depth, split)].join('\n').replace(/\n{3,}/g, '\n\n').trimEnd() + '\n';
+
+/** Every section inside `node` still rendered inline (not split, and not
+ *  inside a split one), with the characters it takes. */
+function inlineSections(node, depth, split, out = []) {
+  for (const section of node.sections) {
+    if (split.has(section)) continue;
+    out.push({ section, size: render([], { items: [], sections: [section] }, depth, split).length });
+    inlineSections(section, depth + 1, split, out);
+  }
+  return out;
+}
+
+/** A section whose own page list alone is too big becomes parts, each a
+ *  sub-section small enough for a file of its own. */
+function chunkItems(node, maxChars, label) {
+  const budget = Math.max(1000, maxChars - 2000);
+  const parts = [];
+  let current = [];
+  let size = 0;
+  for (const p of node.items) {
+    const n = pageLine(p).length + 1;
+    if (current.length && size + n > budget) {
+      parts.push(current);
+      current = [];
+      size = 0;
+    }
+    current.push(p);
+    size += n;
+  }
+  if (current.length) parts.push(current);
+  // One part is the list as it was - nothing smaller to split into (a single
+  // page line longer than the budget). Leave it, rather than loop.
+  if (parts.length < 2) return [];
+  const sections = parts.map((items, i) => ({ label: `${label} (part ${i + 1} of ${parts.length})`, items, sections: [] }));
+  node.sections = [...sections, ...node.sections];
+  node.items = [];
+  return sections;
+}
+
+/** llms.txt, and any child files it needs, as Map<path, text>: 'llms.txt',
+ *  then e.g. 'llms/core-platform.md'. `urlFor(path)` gives a file's URL. */
+export function renderLlmsFiles({ name, description, tree, maxChars, urlFor }) {
+  const files = new Map();
+  const taken = new Set(['llms.txt']);
+  const rootUrl = urlFor('llms.txt');
+
+  function emit(filePath, header, node, depth, trail) {
+    const split = new Map();
+    const splitOut = (section) => {
+      const dir = filePath === 'llms.txt' ? 'llms' : filePath.replace(/\.md$/, '');
+      let childPath = `${dir}/${slugify(section.label)}.md`;
+      for (let n = 2; taken.has(childPath); n++) childPath = `${dir}/${slugify(section.label)}-${n}.md`;
+      taken.add(childPath);
+      split.set(section, { path: childPath, url: urlFor(childPath) });
+    };
+    // A page list too long for this file on its own becomes parts - all of
+    // them separate files, so this one is a plain list of the parts.
+    if (render(header, { items: node.items, sections: [] }, depth, split).length > maxChars) {
+      chunkItems(node, maxChars, trail.length ? trail[trail.length - 1] : 'Pages').forEach(splitOut);
+    }
+    // Then the biggest sections move out, one at a time, until this fits.
+    while (render(header, node, depth, split).length > maxChars) {
+      const candidates = inlineSections(node, depth, split);
+      if (!candidates.length) break;
+      splitOut(candidates.reduce((a, b) => (b.size > a.size ? b : a)).section);
+    }
+    files.set(filePath, render(header, node, depth, split));
+    for (const [section, file] of split) {
+      const childTrail = [...trail, section.label];
+      const childHeader = [`# ${name}: ${childTrail.join(' > ')}`, '', `> Part of the ${name} docs index: ${rootUrl}`, ''];
+      emit(file.path, childHeader, section, 2, childTrail);
+    }
+  }
+
+  const header = [`# ${name}`, ''];
+  if (description) header.push(`> ${description}`, '');
+  emit('llms.txt', header, tree, 2, []);
+  return files;
+}

package/src/lib/pages.js

@@ -61,24 +61,31 @@ function readIgnoreFile(contentDir) {
  *  groups' `page`) - read straight from the file, since this runs before
  *  (and independently of) config validation. Empty when there's no
  *  readable writedocs.json. */
-function navigationPageIds(contentDir) {
-  let config;
-  try {
-    config = JSON.parse(readConfigText(contentDir));
-  } catch {
-    return new Set();
-  }
+/** Every page id a `navigation` lists, in the order a reader meets them -
+ *  top to bottom, each group's own `page` before its `pages` - without
+ *  repeats. */
+export function navigationPageOrder(navigation) {
   const ids = new Set();
   (function walk(node) {
     if (Array.isArray(node)) node.forEach((item) => (typeof item === 'string' ? ids.add(item) : walk(item)));
     else if (node && typeof node === 'object') {
+      if (typeof node.page === 'string') ids.add(node.page);
       for (const [key, value] of Object.entries(node)) {
-        if (key === 'page' && typeof value === 'string') ids.add(value);
-        else if (key !== 'openapi' && key !== 'href') walk(value);
+        if (key !== 'page' && key !== 'openapi' && key !== 'href') walk(value);
       }
     }
-  })(config.navigation);
-  return ids;
+  })(navigation);
+  return [...ids];
+}
+
+function navigationPageIds(contentDir) {
+  let config;
+  try {
+    config = JSON.parse(readConfigText(contentDir));
+  } catch {
+    return new Set();
+  }
+  return new Set(navigationPageOrder(config.navigation));
 }
 
 /** Recursively finds every .md/.mdx file under `contentDir` that is a page:
@@ -198,9 +205,8 @@ export function fileIdForPath(relativePath) {
  *
  *  Both `css`/`js` and `publicCss`/`publicJs` are sorted alphabetically
  *  by their respective path for deterministic load order across rebuilds
- *  - same reasoning as llms.txt's own alphabetical-by-slug sort (see
- *  llms.txt.ts) - filesystem readdir order isn't guaranteed portable
- *  across OSes or directory-walk order otherwise.
+ *  - filesystem readdir order isn't guaranteed portable across OSes or
+ *  directory-walk order otherwise.
  *
  *  `css`/`js` return POSIX-separated paths relative to `contentDir`, not
  *  absolute paths or file contents - BaseLayout.astro (the sole caller)

package/src/pages/[...slug].md.ts

@@ -4,7 +4,7 @@ import path from 'node:path';
 import type { APIRoute } from 'astro';
 import { loadDocsConfig, normalizeEntryId, findAllPages } from '../lib/config';
 import { writedocsTempDir } from '../lib/writedocs-temp-dir.js';
-import { applyVisibilityForAgents } from '../lib/visibility.js';
+import { markdownForAgents } from '../lib/agent-markdown.js';
 
 // The raw-Markdown twin of [...slug].astro: every content page is also
 // reachable at the exact same slug with a literal ".md" suffix (e.g.
@@ -58,25 +58,34 @@ export async function getStaticPaths() {
     .filter((entry) => !entry.data.url)
     .map((entry) => ({
       params: { slug: normalizeEntryId(entry.id) },
-      props: { entry },
+      // Passed along rather than re-read per page - GET only renders one entry.
+      props: { entry, variables: config.variables },
     }));
 }
 
 interface Props {
   entry: DocsEntry;
+  variables: Record<string, string>;
 }
 
 export const GET: APIRoute = ({ props }) => {
-  const { entry } = props as Props;
+  const { entry, variables } = props as Props;
+  const contentDir = process.env.WRITEDOCS_CONTENT_DIR || process.cwd();
+  const packageRoot = process.env.WRITEDOCS_PACKAGE_ROOT || process.cwd();
   // entry.body is the raw, frontmatter-stripped MDX/Markdown source text
   // Astro's glob() loader already read off disk and stashed on every
   // entry by default (see astro/dist/content/loaders/glob.js) - reusing
   // it here means this route needs no separate file read/parse of its
   // own, and always matches exactly what [...slug].astro rendered from
   // (same entry, same collection query).
-  // Mintlify's <Visibility>: this is the agents' view of the page - see
-  // lib/visibility.js.
-  const body = applyVisibilityForAgents(entry.body ?? '');
+  // What the page shows rather than its raw source: Mintlify's <Visibility>
+  // for agents, imported snippets inlined, `variables` filled in - see
+  // lib/agent-markdown.js (llms-full.txt uses the same).
+  const body = markdownForAgents(entry.body ?? '', {
+    file: entry.filePath ? path.resolve(packageRoot, entry.filePath) : undefined,
+    contentDir,
+    variables,
+  });
   const markdown = `# ${entry.data.title}\n\n${body}`;
   return new Response(markdown, {
     headers: { 'Content-Type': 'text/markdown; charset=utf-8' },

package/src/pages/llms/[...path].md.ts

@@ -0,0 +1,23 @@
+import type { APIRoute } from 'astro';
+import { llmsIndexFiles } from '../../lib/llms-index';
+
+// The child files a big site's llms.txt links to - /llms/<section>.md, one
+// per navigation section that didn't fit in llms.txt itself (see
+// lib/llms.js). Most sites have none: llms.txt holds every page, and this
+// route produces nothing. Nor with a hand-written llms.txt, which replaces
+// the generated index as a whole.
+export async function getStaticPaths() {
+  const files = await llmsIndexFiles();
+  if (!files) return [];
+  return [...files.entries()]
+    .filter(([file]) => file.startsWith('llms/'))
+    .map(([file, text]) => ({
+      params: { path: file.replace(/^llms\//, '').replace(/\.md$/, '') },
+      props: { text },
+    }));
+}
+
+export const GET: APIRoute = ({ props }) =>
+  new Response((props as { text: string }).text, {
+    headers: { 'Content-Type': 'text/markdown; charset=utf-8' },
+  });

package/src/pages/llms-full.txt.ts

@@ -2,9 +2,10 @@ import { getCollection, type CollectionEntry } from 'astro:content';
 import fs from 'node:fs';
 import path from 'node:path';
 import type { APIRoute } from 'astro';
-import { loadDocsConfig, normalizeEntryId, findAllPages } from '../lib/config';
+import { loadDocsConfig, normalizeEntryId, findAllPages, resolveSiteUrl, fileIdForEntry } from '../lib/config';
 import { writedocsTempDir } from '../lib/writedocs-temp-dir.js';
-import { applyVisibilityForAgents } from '../lib/visibility.js';
+import { markdownForAgents } from '../lib/agent-markdown.js';
+import { orderByNavigation } from '../lib/llms.js';
 
 // The "everything, concatenated" half of the llms.txt pair - see
 // llms.txt.ts (right next to this file) for the lightweight index half,
@@ -29,7 +30,9 @@ export const GET: APIRoute = async () => {
     });
   }
 
+  const packageRoot = process.env.WRITEDOCS_PACKAGE_ROOT || process.cwd();
   const config = loadDocsConfig(contentDir);
+  const siteUrl = resolveSiteUrl(config);
 
   const hasGeneratedDocs = fs.existsSync(path.join(writedocsTempDir(contentDir), 'generated-docs'));
   const hasPages = findAllPages(contentDir).length > 0;
@@ -39,7 +42,7 @@ export const GET: APIRoute = async () => {
   ]);
   const entries: DocsEntry[] = [...pagesEntries, ...generatedDocsEntries];
 
-  const sections = entries
+  const unordered = entries
     // Same exclusion [...slug].md.ts applies to its own per-page raw
     // Markdown route, for the same reason: an OpenAPI operation page
     // (generated stub, or a hand-written page that opts into rendering
@@ -54,17 +57,35 @@ export const GET: APIRoute = async () => {
     .filter((entry) => !entry.data.url)
     .map((entry: DocsEntry) => {
       const slug = normalizeEntryId(entry.id);
-      // Agents' view of <Visibility> blocks, same as the .md route.
-      const body = applyVisibilityForAgents(entry.body ?? '');
-      return { slug, title: entry.data.title, body };
-    })
-    .sort((a, b) => a.slug.localeCompare(b.slug));
+      // What the page shows, not its raw source: <Visibility> for agents,
+      // imported snippets inlined, `variables` filled in - same as the .md
+      // route (lib/agent-markdown.js).
+      const body = markdownForAgents(entry.body ?? '', {
+        file: entry.filePath ? path.resolve(packageRoot, entry.filePath) : undefined,
+        contentDir,
+        variables: config.variables,
+      });
+      // The page's own URL, so an agent can cite it.
+      const url = `${siteUrl ?? ''}${slug === 'index' ? '/' : `/${slug}/`}`;
+      return {
+        slug,
+        fileId: fileIdForEntry(contentDir, packageRoot, entry),
+        title: entry.data.title,
+        description: entry.data.description,
+        url,
+        body,
+      };
+    });
+  // Navigation order, same as llms.txt.
+  const sections = orderByNavigation(unordered, config.navigation);
 
   const lines: string[] = [`# ${config.name}`, ''];
   if (config.description) lines.push(`> ${config.description}`, '');
 
   for (const section of sections) {
-    lines.push('---', '', `# ${section.title}`, '', section.body.trim(), '');
+    lines.push('---', '', `# ${section.title}`, '', `URL: ${section.url}`, '');
+    if (section.description) lines.push(`> ${section.description}`, '');
+    lines.push(section.body.trim(), '');
   }
 
   const content = lines.join('\n') + '\n';

package/src/pages/llms.txt.ts

@@ -1,9 +1,7 @@
-import { getCollection, type CollectionEntry } from 'astro:content';
 import fs from 'node:fs';
 import path from 'node:path';
 import type { APIRoute } from 'astro';
-import { loadDocsConfig, normalizeEntryId, findAllPages, resolveSiteUrl } from '../lib/config';
-import { writedocsTempDir } from '../lib/writedocs-temp-dir.js';
+import { llmsIndexFiles } from '../lib/llms-index';
 
 // A lightweight, LLM-facing index of every page on the site - the
 // llms.txt convention (see https://llmstxt.org), also adopted by Mintlify
@@ -13,136 +11,30 @@ import { writedocsTempDir } from '../lib/writedocs-temp-dir.js';
 // of the pair; llms-full.txt (llms-full.txt.ts, right next to this file)
 // is the "everything, concatenated" half.
 //
-// Fixed, non-dynamic route (no getStaticPaths/params needed) - Astro
-// prerenders this once at the literal path /llms.txt, same as any other
-// static-output route (this project's astro.config.mjs sets
-// `output: 'static'`), and re-evaluates it per request in `writedocs dev`
-// - see [...slug].md.ts for the same fixed-route-vs-catchall distinction
-// applied to per-page raw Markdown.
+// Pages are listed under headings that follow the navigation. A site too
+// big for one file keeps this as a directory whose largest sections link to
+// child files - llms/[...path].md.ts serves those - so no page is left out;
+// see lib/llms.js.
 //
-// Unconditional - unlike the per-page .md routes in [...slug].md.ts,
-// this doesn't depend on writedocs.json's `contextMenu` field. There's no UI
+// Fixed, non-dynamic route - Astro prerenders this once at the literal path
+// /llms.txt (this project's astro.config.mjs sets `output: 'static'`), and
+// re-evaluates it per request in `writedocs dev`.
+//
+// Unconditional - unlike the per-page .md routes in [...slug].md.ts, this
+// doesn't depend on writedocs.json's `contextMenu` field. There's no UI
 // footprint to gate (it's an extra static file, not a visible menu), so
 // it's always generated. `contextMenu` only affects which URL each entry
-// links to below (a real fetchable .md URL if the per-page raw-Markdown
-// routes exist, the ordinary HTML page URL otherwise).
-type DocsEntry = CollectionEntry<'pages'> | CollectionEntry<'generatedDocs'>;
-
-// Mirrors Mintlify's own llms.txt behavior: truncate a page's frontmatter
-// `description` at the first line break (a multi-paragraph description
-// would blow out a one-line list entry) and at 300 characters (an
-// arbitrary but reasonable cap - keeps every entry scannable regardless
-// of how verbose an individual page's description happens to be).
-const DESCRIPTION_MAX_CHARS = 300;
-function truncateDescription(description: string | undefined): string | undefined {
-  if (!description) return undefined;
-  const firstLine = description.split('\n')[0].trim();
-  if (!firstLine) return undefined;
-  return firstLine.length > DESCRIPTION_MAX_CHARS
-    ? firstLine.slice(0, DESCRIPTION_MAX_CHARS).trimEnd() + '…'
-    : firstLine;
-}
-
-// Same cap Mintlify documents for its own auto-generated llms.txt - this
-// file is meant to stay a lightweight index (one line per page), not
-// balloon into something llms-full.txt-sized. Realistically unlikely to
-// matter for most Writedocs sites (a site would need on the order of a
-// thousand-plus pages with full-length descriptions to hit this), but
-// cheap to guard against regardless.
-const MAX_CHARS = 100_000;
-
+// links to (a real fetchable .md URL if the per-page raw-Markdown routes
+// exist, the ordinary HTML page URL otherwise).
 export const GET: APIRoute = async () => {
-  const contentDir = process.env.WRITEDOCS_CONTENT_DIR || process.cwd();
-
-  // A hand-authored llms.txt at the project root (next to writedocs.json)
-  // always wins outright over the generated one below - same override
-  // convention Mintlify documents for its own llms.txt. Lets a site
-  // author hand-curate this file (different ordering, editorial
-  // descriptions, extra sections) without losing the ability to fall
-  // back to the auto-generated version by just deleting the override.
-  const customPath = path.join(contentDir, 'llms.txt');
-  if (fs.existsSync(customPath)) {
-    return new Response(fs.readFileSync(customPath, 'utf-8'), {
-      headers: { 'Content-Type': 'text/plain; charset=utf-8' },
-    });
-  }
-
-  const config = loadDocsConfig(contentDir);
-  const siteUrl = resolveSiteUrl(config); // absolute origin if `domain` is set, else null
-
-  const hasGeneratedDocs = fs.existsSync(path.join(writedocsTempDir(contentDir), 'generated-docs'));
-  const hasPages = findAllPages(contentDir).length > 0;
-  const [pagesEntries, generatedDocsEntries] = await Promise.all([
-    hasPages ? getCollection('pages') : Promise.resolve([]),
-    hasGeneratedDocs ? getCollection('generatedDocs') : Promise.resolve([]),
-  ]);
-  const entries: DocsEntry[] = [...pagesEntries, ...generatedDocsEntries];
-
-  const items = entries
-    // A page that opted out of search-engine indexing (frontmatter
-    // `seo.noindex: true`, already excluded from sitemap.xml - see
-    // astro.config.mjs's collectNoindexIds()) is excluded here for the
-    // same reason: llms.txt exists to help external tools discover
-    // pages, exactly what `noindex` asked not to happen.
-    .filter((entry) => !entry.data.seo?.noindex)
-    // A frontmatter `url` page (Mintlify's external link) has no content -
-    // its own URL only redirects to the link.
-    .filter((entry) => !entry.data.url)
-    .map((entry: DocsEntry) => {
-      const slug = normalizeEntryId(entry.id);
-      // Same URL Astro's own router resolves this entry to - see
-      // hrefForSlug() in [...slug].astro for the HTML form, and
-      // [...slug].md.ts for the .md form (no trailing slash, "index.md"
-      // for the home page rather than the HTML convention's bare "/").
-      // An OpenAPI operation page has no .md form ([...slug].md.ts leaves
-      // it out - it renders from the spec, not prose), so it keeps its HTML
-      // URL even when the .md routes exist.
-      const hasMarkdownRoute = config.contextMenu && !entry.data.openapi;
-      const path_ = hasMarkdownRoute ? `/${slug}.md` : slug === 'index' ? '/' : `/${slug}/`;
-      const href = (siteUrl ?? '') + path_;
-      let description = truncateDescription(entry.data.description);
-      // Mirrors Mintlify's own behavior: an OpenAPI operation page's
-      // description gets its "METHOD /path" appended, since the page
-      // itself renders almost entirely from the spec at request time
-      // (see ApiPlayground.astro) rather than from frontmatter prose.
-      if (entry.data.openapi) {
-        description = description ? `${description} (${entry.data.openapi})` : entry.data.openapi;
-      }
-      return { title: entry.data.title, href, description, slug };
-    })
-    .sort((a, b) => a.slug.localeCompare(b.slug));
-
-  const lines: string[] = [`# ${config.name}`, ''];
-  if (config.description) lines.push(`> ${config.description}`, '');
-  lines.push('## Docs', '');
-  for (const item of items) {
-    const entryLine = `- [${item.title}](${item.href})${item.description ? `: ${item.description}` : ''}`;
-    lines.push(entryLine);
-  }
-
-  let content = lines.join('\n') + '\n';
-  if (content.length > MAX_CHARS) {
-    // Truncate at the last full line inside the budget, then note how
-    // many entries got cut - rather than silently producing a
-    // mid-sentence-cutoff file, or (worse) one JSON/Markdown-breaking
-    // half-written link.
-    const truncatedLines = lines.slice(0, 4 + (config.description ? 2 : 0)); // "# name" / "" / ["> desc" / ""] / "## Docs" / ""
-    let runningLength = truncatedLines.join('\n').length + 1;
-    let omitted = 0;
-    for (const item of items) {
-      const entryLine = `- [${item.title}](${item.href})${item.description ? `: ${item.description}` : ''}`;
-      if (runningLength + entryLine.length + 1 > MAX_CHARS - 200) {
-        omitted++;
-        continue;
-      }
-      truncatedLines.push(entryLine);
-      runningLength += entryLine.length + 1;
-    }
-    truncatedLines.push('', `_Truncated — ${omitted} more page(s) omitted. See llms-full.txt or the site's own navigation for the complete list._`);
-    content = truncatedLines.join('\n') + '\n';
-  }
-
-  return new Response(content, {
+  const files = await llmsIndexFiles();
+  // null: the project has its own llms.txt at its root, next to
+  // writedocs.json - it replaces the generated one (and its child files)
+  // outright, and a site can fall back by just deleting it.
+  const text = files
+    ? files.get('llms.txt')
+    : fs.readFileSync(path.join(process.env.WRITEDOCS_CONTENT_DIR || process.cwd(), 'llms.txt'), 'utf-8');
+  return new Response(text, {
     headers: { 'Content-Type': 'text/plain; charset=utf-8' },
   });
 };