@writedocs/generator 0.7.3 → 0.8.0

package/astro.config.mjs

@@ -36,6 +36,7 @@ import { remarkUnknownComponentFallback } from './src/lib/mdx-unknown-components
 import { remarkExtractInlineReactComponents } from './src/lib/mdx-inline-react.js';
 import { writedocsTempDir, writedocsBuildStagingDir } from './src/lib/writedocs-temp-dir.js';
 import { stylesAssetFallback } from './src/lib/styles-asset-integration.js';
+import { mcpDevServer } from './src/lib/mcp-dev-integration.js';
 import { report } from './src/lib/cli-report.js';
 import {
   loadDocsConfig,
@@ -309,6 +310,9 @@ export default defineConfig({
     // field list and styles-asset-integration.js for how it's actually
     // served/copied.
     stylesAssetFallback(contentDir),
+    // /mcp in `writedocs dev`, as the build's dist/_worker.js serves it -
+    // see src/lib/mcp-dev-integration.js.
+    mcpDevServer({ version: JSON.parse(fs.readFileSync(path.join(packageRoot, 'package.json'), 'utf-8')).version }),
   ],
   output: 'static',
   // Dual Shiki themes for fenced code blocks (```) in MDX content, so

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@writedocs/generator",
-  "version": "0.7.3",
+  "version": "0.8.0",
   "description": "Static site generator for docs — a writedocs.json + MDX folder in, a static site out.",
   "type": "module",
   "bin": {
@@ -9,7 +9,8 @@
   "exports": {
     "./components": "./src/components/index.ts",
     "./config-schema": "./src/lib/config-schema.js",
-    "./writedocs.schema.json": "./writedocs.schema.json"
+    "./writedocs.schema.json": "./writedocs.schema.json",
+    "./mcp": "./src/mcp/server.js"
   },
   "files": [
     "bin",

package/src/cli/build.js

@@ -5,6 +5,7 @@ import { runPagefind } from './run-pagefind.js';
 import { preflightCheck, sameDriveCheck, writableInstallCheck } from './preflight.js';
 import { generateApiPages } from './generate-api-pages.js';
 import { writeRedirectsFile } from './write-redirects-file.js';
+import { writeMcpFiles } from './write-mcp-files.js';
 import { writedocsBuildStagingDir } from '../lib/writedocs-temp-dir.js';
 import { log, step, plural, duration, displayPath, formatProblems, color, CliExit } from './output.js';
 import { describeError, builtRoute, authorWarning, verboseLine } from './astro-output.js';
@@ -112,6 +113,13 @@ export async function runBuild({ contentDir, packageRoot, verbose = false }) {
   // - this turns those into real instant edge redirects on hosts that read
   // a `_redirects` file (Cloudflare Pages, Netlify), purely additively.
   writeRedirectsFile(distDir, contentDir);
+  // The site's MCP server at /mcp: mcp-index.json is already in dist/ (an
+  // Astro route); this adds the Cloudflare _worker.js that serves it - see
+  // write-mcp-files.js for every host.
+  const { version: writedocsVersion } = JSON.parse(fs.readFileSync(path.join(packageRoot, 'package.json'), 'utf-8'));
+  const mcp = writeMcpFiles(distDir, contentDir, writedocsVersion);
+  if (mcp.written.length) notes.push('MCP server at /mcp - dist/_worker.js runs it on Cloudflare; mcp-index.json holds the pages.');
+  if (mcp.skipped) addWarning(null, `No _worker.js for the MCP server: ${mcp.skipped}.`);
 
   const indexing = step('Indexing search');
   try {

package/src/cli/write-mcp-files.js

@@ -0,0 +1,96 @@
+// After `writedocs build`: the files that make dist/ serve the site's MCP
+// server at /mcp on Cloudflare, with no setup of the host's own - next to
+// mcp-index.json, which the build itself writes (src/pages/[mcpIndex].json.ts).
+//
+//   _worker.js     the MCP server (src/mcp/server.js, inlined - no imports)
+//                  plus a small fetch handler: /mcp goes to the server,
+//                  everything else to the static files (env.ASSETS).
+//                  - Cloudflare Pages runs it on its own ("advanced mode").
+//                  - Cloudflare Workers with static assets: point `main` at
+//                    dist/_worker.js, assets at dist/ with an ASSETS binding.
+//   _routes.json   Pages: only /mcp runs the worker; every other path stays a
+//                  plain static file (and `_redirects` keeps applying).
+//   .assetsignore  Workers: don't publish _worker.js / _routes.json as files.
+//
+// Anywhere else - Netlify, Vercel, the WriteDocs platform's serving Worker -
+// import handleMcpHttp from `@writedocs/generator/mcp` and give it
+// mcp-index.json; a purely static host (GitHub Pages, S3) can't run /mcp.
+//
+// A project that brings its own _worker.js or _routes.json (in public/), or a
+// Pages `functions/` folder (which Pages ignores once a _worker.js exists),
+// keeps them: the worker isn't written, and the build says so.
+import fs from 'node:fs';
+import path from 'node:path';
+import { fileURLToPath } from 'node:url';
+
+const SERVER_SOURCE = path.join(path.dirname(fileURLToPath(import.meta.url)), '..', 'mcp', 'server.js');
+const IGNORED_BY_WORKERS = ['_worker.js', '_routes.json'];
+
+/** The `_worker.js` source: server.js without its `export`s, and the fetch
+ *  handler. `version` is the writedocs version, reported by the server. */
+export function mcpWorkerSource(version) {
+  const server = fs.readFileSync(SERVER_SOURCE, 'utf-8').replace(/^export (?=(?:const|let|async function|function) )/gm, '');
+  return `// Generated by writedocs ${version} - the site's MCP server at /mcp.
+// Cloudflare Pages runs this file on its own; on Cloudflare Workers, use it as
+// \`main\` with dist/ as the static assets (binding ASSETS). See
+// https://github.com/writedocs/writedocs (src/cli/write-mcp-files.js).
+
+${server}
+const WRITEDOCS_VERSION = ${JSON.stringify(version)};
+
+// The page index, read once per isolate from the site's own static files.
+let indexPromise = null;
+function loadIndex(request, env) {
+  if (!indexPromise) {
+    indexPromise = env.ASSETS.fetch(new URL('/mcp-index.json', request.url))
+      .then((res) => {
+        if (!res.ok) throw new Error('mcp-index.json: HTTP ' + res.status);
+        return res.json();
+      })
+      .catch((err) => {
+        indexPromise = null;
+        throw err;
+      });
+  }
+  return indexPromise;
+}
+
+export default {
+  async fetch(request, env) {
+    const { pathname } = new URL(request.url);
+    if (pathname === '/mcp' || pathname === '/mcp/') {
+      return handleMcpHttp(request, { loadIndex: () => loadIndex(request, env), version: WRITEDOCS_VERSION });
+    }
+    return env.ASSETS.fetch(request);
+  },
+};
+`;
+}
+
+/** Writes the files into `distDir`. Returns { written: [names], skipped:
+ *  reason or null } - nothing at all when there's no mcp-index.json
+ *  (writedocs.json `"mcp": false`). */
+export function writeMcpFiles(distDir, contentDir, version) {
+  if (!fs.existsSync(path.join(distDir, 'mcp-index.json'))) return { written: [], skipped: null };
+
+  const own = ['_worker.js', '_routes.json'].filter((name) => fs.existsSync(path.join(distDir, name)));
+  if (own.length) {
+    return { written: [], skipped: `the project has its own ${own.join(' and ')} - /mcp needs its handler added there (see @writedocs/generator/mcp)` };
+  }
+  if (fs.existsSync(path.join(contentDir, 'functions'))) {
+    return { written: [], skipped: 'the project has a Cloudflare Pages functions/ folder, which a _worker.js would switch off - add /mcp there with @writedocs/generator/mcp' };
+  }
+
+  fs.writeFileSync(path.join(distDir, '_worker.js'), mcpWorkerSource(version));
+  fs.writeFileSync(path.join(distDir, '_routes.json'), JSON.stringify({ version: 1, include: ['/mcp', '/mcp/'], exclude: [] }, null, 2) + '\n');
+
+  const ignorePath = path.join(distDir, '.assetsignore');
+  const existing = fs.existsSync(ignorePath) ? fs.readFileSync(ignorePath, 'utf-8') : '';
+  const lines = existing.split(/\r?\n/).map((l) => l.trim());
+  const missing = IGNORED_BY_WORKERS.filter((name) => !lines.includes(name));
+  if (missing.length) {
+    const prefix = existing && !existing.endsWith('\n') ? `${existing}\n` : existing;
+    fs.writeFileSync(ignorePath, `${prefix}${missing.join('\n')}\n`);
+  }
+  return { written: ['_worker.js', '_routes.json', '.assetsignore'], skipped: null };
+}

package/src/lib/agent-markdown.js

@@ -0,0 +1,142 @@
+// The Markdown an AI agent reads for a page - llms-full.txt and the per-page
+// .md route. It starts from the page's raw MDX, so everything the build does
+// to that source on the way to HTML has to be redone here, or the agent reads
+// something other than the page: `<Visibility>` blocks, imported .mdx
+// snippets (inlined, as the page shows them) and writedocs.json `variables`.
+//
+// Code is left exactly as written - fenced blocks and inline `code` - the same
+// way the build never substitutes inside code: a page documenting the
+// `[[key]]` syntax, or showing `<Snippet />` in an example, must keep it.
+// Plain JavaScript, so it can be tested with plain Node.
+import fs from 'node:fs';
+import path from 'node:path';
+import matter from 'gray-matter';
+import { applyVisibilityForAgents } from './visibility.js';
+
+/** `text` with `transform` applied to everything outside fenced code blocks
+ *  and inline code spans; code comes back untouched. */
+export function mapOutsideCode(text, transform) {
+  const out = [];
+  let prose = [];
+  let fence = null; // the opening marker (``` or ~~~, maybe longer) while inside a block
+  const flush = () => {
+    if (prose.length) out.push(mapOutsideInlineCode(prose.join('\n'), transform));
+    prose = [];
+  };
+  for (const line of text.split('\n')) {
+    const marker = /^\s*(`{3,}|~{3,})/.exec(line)?.[1];
+    if (fence) {
+      out.push(line);
+      if (marker && marker[0] === fence[0] && marker.length >= fence.length && line.trim() === marker) fence = null;
+    } else if (marker) {
+      flush();
+      fence = marker;
+      out.push(line);
+    } else {
+      prose.push(line);
+    }
+  }
+  flush();
+  return out.join('\n');
+}
+
+function mapOutsideInlineCode(text, transform) {
+  // Backtick runs of equal length delimit a code span (CommonMark).
+  return text
+    .split(/(`+)([\s\S]*?)(\1)(?!`)/)
+    .reduce((acc, part, i, parts) => {
+      const k = i % 4;
+      if (k === 0) acc.push(transform(part));
+      else if (k === 1) acc.push(part + parts[i + 1] + parts[i + 2]);
+      return acc;
+    }, [])
+    .join('');
+}
+
+const PLACEHOLDER = /\[\[\s*([\w.-]+)\s*\]\]/g;
+// A standalone `{key}` or `{{key}}` expression (Mintlify writes the latter;
+// MDX reads it as `{key}` inside an expression) - not an attribute value
+// (`prop={key}`), which the build leaves alone too. The double form first,
+// or its outer braces would be left around the value.
+const MINTLIFY_PLACEHOLDER = /(?<!=\s*)\{\s*\{\s*([A-Za-z_$][\w$]*)\s*\}\s*\}|(?<!=\s*)\{\s*([A-Za-z_$][\w$]*)\s*\}/g;
+
+/** writedocs.json `variables` - `[[key]]`, and Mintlify's `{key}` - replaced
+ *  the way the build replaces them (lib/mdx-substitute-variables.js): outside
+ *  code, and only for keys that exist. */
+export function substituteVariables(text, variables) {
+  if (!variables || Object.keys(variables).length === 0) return text;
+  const has = (key) => Object.prototype.hasOwnProperty.call(variables, key);
+  return mapOutsideCode(text, (prose) =>
+    prose
+      .replace(PLACEHOLDER, (match, key) => (has(key) ? String(variables[key]) : match))
+      .replace(MINTLIFY_PLACEHOLDER, (match, doubled, single) => {
+        const key = doubled ?? single;
+        return has(key) ? String(variables[key]) : match;
+      })
+  );
+}
+
+const IMPORT_LINE = /^[ \t]*import\s+([A-Za-z_$][\w$]*)\s+from\s+['"]([^'"]+\.mdx?)['"];?[ \t]*$/gm;
+
+function resolveSnippet(spec, fromFile, contentDir) {
+  if (spec.startsWith('/snippets/')) return path.join(contentDir, 'snippets', spec.slice('/snippets/'.length));
+  if (spec.startsWith('./') || spec.startsWith('../')) return path.resolve(path.dirname(fromFile), spec);
+  if (spec.startsWith('/')) return path.join(contentDir, spec.slice(1));
+  return null;
+}
+
+function attributesOf(source) {
+  const props = {};
+  for (const m of source.matchAll(/([A-Za-z_$][\w$-]*)\s*=\s*(?:"([^"]*)"|'([^']*)'|\{\s*["']([^"']*)["']\s*\})/g)) {
+    props[m[1]] = m[2] ?? m[3] ?? m[4];
+  }
+  return props;
+}
+
+function fillProps(snippet, props, children) {
+  return snippet
+    .replace(/\{\s*props\.children\s*\}/g, children ?? '')
+    .replace(/\{\s*props\.([A-Za-z_$][\w$]*)\s*\}/g, (match, key) => (key in props ? props[key] : match));
+}
+
+/** Imported .mdx snippets inlined where the page uses them - the snippet's
+ *  own text, with `{props.x}` filled from the tag's string attributes, and
+ *  its own imports inlined in turn. The import line goes too. A snippet that
+ *  can't be read, or a .jsx/.tsx component, stays as written. */
+export function inlineSnippets(text, { file, contentDir, depth = 0 }) {
+  if (depth > 5) return text;
+  const snippets = new Map();
+  let result = mapOutsideCode(text, (prose) =>
+    prose.replace(IMPORT_LINE, (line, name, spec) => {
+      const target = resolveSnippet(spec, file, contentDir);
+      let raw;
+      try {
+        raw = target && fs.readFileSync(target, 'utf-8');
+      } catch {
+        raw = null;
+      }
+      if (!raw) return line;
+      const body = inlineSnippets(matter(raw).content.trim(), { file: target, contentDir, depth: depth + 1 });
+      snippets.set(name, body);
+      return '';
+    })
+  );
+  for (const [name, body] of snippets) {
+    result = mapOutsideCode(result, (prose) =>
+      prose
+        .replace(new RegExp(`<${name}\\b([^>]*?)\\/>`, 'g'), (_, attrs) => fillProps(body, attributesOf(attrs)))
+        .replace(new RegExp(`<${name}\\b([^>]*)>([\\s\\S]*?)<\\/${name}>`, 'g'), (_, attrs, children) =>
+          fillProps(body, attributesOf(attrs), children.trim())
+        )
+    );
+  }
+  return result.replace(/\n{3,}/g, '\n\n');
+}
+
+/** A page's body as an agent should read it. `file` is the page's absolute
+ *  path (to resolve relative snippet imports). */
+export function markdownForAgents(body, { file, contentDir, variables }) {
+  let text = applyVisibilityForAgents(body ?? '');
+  if (file) text = inlineSnippets(text, { file, contentDir });
+  return substituteVariables(text, variables);
+}

package/src/lib/config-file.js

@@ -4,7 +4,7 @@
 //
 // Windows editors - Notepad, PowerShell 5.1's `Out-File`/`Set-Content
 // -Encoding utf8` - save UTF-8 with a byte order mark. JSON.parse rejects
-// it ("Unexpected token ''"), and the error then blames commas and
+// it ("Unexpected token U+FEFF"), and the error then blames commas and
 // brackets the file doesn't have. Every reader strips it here instead.
 import fs from 'node:fs';
 import path from 'node:path';

package/src/lib/config-schema.js

@@ -522,6 +522,11 @@ const docsConfigSchema = z.object({
   // The "Copy page" dropdown - see contextMenuSchema above. Absent by
   // default (no menu, no .md routes).
   contextMenu: contextMenuSchema.optional(),
+  // The site's MCP server - an AI tool connects to `/mcp` and searches and
+  // reads the docs. On by default: `writedocs build` writes the page index
+  // (mcp-index.json) and a Cloudflare-ready `_worker.js` into dist/ (see
+  // src/cli/write-mcp-files.js). `false` leaves all of it out.
+  mcp: z.boolean().default(true),
   // See redirectSchema above. Wired directly into Astro's own `redirects`
   // config option in astro.config.mjs.
   redirects: z.array(redirectSchema).default([]),

package/src/lib/config-schema.ts

@@ -999,6 +999,11 @@ export const docsConfigSchema = z.object({
   // The "Copy page" dropdown - see contextMenuSchema above. Absent by
   // default (no menu, no .md routes).
   contextMenu: contextMenuSchema.optional(),
+  // The site's MCP server - an AI tool connects to `/mcp` and searches and
+  // reads the docs. On by default: `writedocs build` writes the page index
+  // (mcp-index.json) and a Cloudflare-ready `_worker.js` into dist/ (see
+  // src/cli/write-mcp-files.js). `false` leaves all of it out.
+  mcp: z.boolean().default(true),
   // See redirectSchema above. Wired directly into Astro's own `redirects`
   // config option in astro.config.mjs.
   redirects: z.array(redirectSchema).default([]),

package/src/lib/json-schema-descriptions.js

@@ -166,6 +166,7 @@ export const DESCRIPTIONS = {
   'seo.noindex': 'Ask search engines not to index pages, and leave them out of sitemap.xml.',
   contextMenu: 'Adds a "Copy page" menu to pages - copy as Markdown, or open the page in an AI assistant - and a Markdown copy of each page at its address + ".md".',
   'contextMenu.openIn': 'Which AI assistants the menu offers. Default: all three.',
+  mcp: 'An MCP server at /mcp, so AI tools can search and read the docs. The build adds its index and a Cloudflare-ready _worker.js to dist/. Default true; false leaves them out.',
   redirects: 'Redirects from old addresses. Each matches one exact path.',
   'redirects[].source': 'The old path, like "/old-page".',
   'redirects[].destination': 'Where to send it, like "/docs/new-page/".',

package/src/lib/llms-index.ts

@@ -0,0 +1,88 @@
+// llms.txt and its child files (/llms/*.md) - the pages an AI tool can
+// discover, in the site's navigation structure. Shared by llms.txt.ts (the
+// root file) and llms/[...path].md.ts (the child files a big site's llms.txt
+// links to), so both render from the same tree in the same run. The
+// structure and the splitting live in lib/llms.js.
+import { getCollection, type CollectionEntry } from 'astro:content';
+import fs from 'node:fs';
+import path from 'node:path';
+import { loadDocsConfig, normalizeEntryId, findAllPages, resolveSiteUrl, fileIdForEntry } from './config';
+import { buildLlmsTree, renderLlmsFiles } from './llms.js';
+import { navigationPageOrder } from './pages.js';
+import { writedocsTempDir } from './writedocs-temp-dir.js';
+
+type DocsEntry = CollectionEntry<'pages'> | CollectionEntry<'generatedDocs'>;
+
+// Mirrors Mintlify's own llms.txt behavior: a page's frontmatter
+// `description` is cut at the first line break (a multi-paragraph
+// description would blow out a one-line list entry) and at 300 characters,
+// so every entry stays scannable.
+const DESCRIPTION_MAX_CHARS = 300;
+function truncateDescription(description: string | undefined): string | undefined {
+  if (!description) return undefined;
+  const firstLine = description.split('\n')[0].trim();
+  if (!firstLine) return undefined;
+  return firstLine.length > DESCRIPTION_MAX_CHARS ? firstLine.slice(0, DESCRIPTION_MAX_CHARS).trimEnd() + '…' : firstLine;
+}
+
+// Most characters one file holds - llms.txt or a child file. The llms.txt
+// spec sets no limit ("small enough to fit in context"); this is Mintlify's,
+// about 25,000 tokens. Past it, sections move into child files instead of
+// pages being dropped.
+export const LLMS_MAX_CHARS = 100_000;
+
+/** The generated files as Map<path, text> - 'llms.txt', then 'llms/<...>.md'
+ *  when the site needs them - or null when the project has its own
+ *  llms.txt, which replaces all of it. */
+export async function llmsIndexFiles(): Promise<Map<string, string> | null> {
+  const contentDir = process.env.WRITEDOCS_CONTENT_DIR || process.cwd();
+  const packageRoot = process.env.WRITEDOCS_PACKAGE_ROOT || process.cwd();
+  // A hand-authored llms.txt at the project root (next to writedocs.json)
+  // wins outright - same override convention Mintlify documents.
+  if (fs.existsSync(path.join(contentDir, 'llms.txt'))) return null;
+
+  const config = loadDocsConfig(contentDir);
+  const siteUrl = resolveSiteUrl(config); // absolute origin if `domain` is set, else null
+
+  const hasGeneratedDocs = fs.existsSync(path.join(writedocsTempDir(contentDir), 'generated-docs'));
+  const hasPages = findAllPages(contentDir).length > 0;
+  const [pagesEntries, generatedDocsEntries] = await Promise.all([
+    hasPages ? getCollection('pages') : Promise.resolve([]),
+    hasGeneratedDocs ? getCollection('generatedDocs') : Promise.resolve([]),
+  ]);
+  const entries: DocsEntry[] = [...pagesEntries, ...generatedDocsEntries];
+
+  const pages = new Map<string, { title: string; href: string; description?: string; slug: string }>();
+  for (const entry of entries) {
+    // A page that opted out of search-engine indexing (`seo.noindex`, also
+    // left out of sitemap.xml) is left out here too, and a frontmatter `url`
+    // page (Mintlify's external link) has no content of its own.
+    if (entry.data.seo?.noindex || entry.data.url) continue;
+    const slug = normalizeEntryId(entry.id);
+    // The .md route when it exists ([...slug].md.ts - with `contextMenu`,
+    // and never for an OpenAPI page, which renders from the spec), else the
+    // HTML page.
+    const hasMarkdownRoute = config.contextMenu && !entry.data.openapi;
+    const href = (siteUrl ?? '') + (hasMarkdownRoute ? `/${slug}.md` : slug === 'index' ? '/' : `/${slug}/`);
+    let description = truncateDescription(entry.data.description);
+    // Mirrors Mintlify: an OpenAPI operation page's description gets its
+    // "METHOD /path" appended - the page itself renders from the spec.
+    if (entry.data.openapi) description = description ? `${description} (${entry.data.openapi})` : entry.data.openapi;
+    pages.set(fileIdForEntry(contentDir, packageRoot, entry), { title: entry.data.title, href, description, slug });
+  }
+
+  const listed = new Set(navigationPageOrder(config.navigation));
+  const unlisted = [...pages.entries()]
+    .filter(([id]) => !listed.has(id))
+    .sort((a, b) => a[1].slug.localeCompare(b[1].slug))
+    .map(([id]) => id);
+  const tree = buildLlmsTree(config.navigation, (id: string) => pages.get(id) ?? null, unlisted);
+
+  return renderLlmsFiles({
+    name: config.name,
+    description: config.description,
+    tree,
+    maxChars: LLMS_MAX_CHARS,
+    urlFor: (file: string) => `${siteUrl ?? ''}/${file}`,
+  });
+}

package/src/lib/llms.js

@@ -0,0 +1,200 @@
+// Shared by the llms routes (lib/llms-index.ts, llms-full.txt.ts): the order
+// pages appear in, and llms.txt's structure. Plain JavaScript, so it can be
+// tested with plain Node.
+import { navigationPageOrder } from './pages.js';
+
+/** `items` ({ fileId, slug }) in navigation order - the order a reader meets
+ *  the pages in the sidebar - then every page the navigation doesn't list,
+ *  by URL. */
+export function orderByNavigation(items, navigation) {
+  const rank = new Map(navigationPageOrder(navigation).map((id, i) => [id, i]));
+  const at = (item) => rank.get(item.fileId) ?? Infinity;
+  return [...items].sort((a, b) => at(a) - at(b) || a.slug.localeCompare(b.slug));
+}
+
+// --- llms.txt as a tree -------------------------------------------------
+//
+// llms.txt lists every page under headings that follow the navigation -
+// products, versions, languages, tabs, dropdowns, groups. A site too big for
+// one file (MAX_CHARS) keeps llms.txt as a directory: its largest sections
+// move into child files under /llms/, linked from where they were, and a
+// child too big is split the same way. No page is ever left out - the scheme
+// Mintlify moved to from cutting the list off
+// (https://www.mintlify.com/blog/scaling-llms-txt).
+
+const CONTAINERS = [
+  ['tabs', 'tab'],
+  ['versions', 'version'],
+  ['languages', 'language'],
+  ['dropdowns', 'dropdown'],
+  ['products', 'product'],
+];
+
+/** A Section: { label, items: [page], sections: [Section] }. `pageFor(id)`
+ *  gives a listed page ({ title, href, description }) or null for one that
+ *  isn't listed (noindex, an external `url`, missing). A page shows up once,
+ *  where the navigation first lists it; `unlisted` pages (not in the
+ *  navigation at all) go in a trailing "Other pages" section. */
+export function buildLlmsTree(navigation, pageFor, unlisted = []) {
+  const seen = new Set();
+  const page = (id) => {
+    if (seen.has(id)) return null;
+    const found = pageFor(id);
+    if (!found) return null;
+    seen.add(id);
+    return found;
+  };
+  const prune = (section) => section.items.length > 0 || section.sections.length > 0;
+
+  function fromList(list) {
+    const node = { items: [], sections: [] };
+    for (const item of list ?? []) {
+      if (typeof item === 'string') {
+        const p = page(item);
+        if (p) node.items.push(p);
+      } else if (item && typeof item === 'object' && typeof item.group === 'string' && Array.isArray(item.pages)) {
+        const own = typeof item.page === 'string' ? page(item.page) : null;
+        const inner = fromList(item.pages);
+        const section = { label: item.group, items: own ? [own, ...inner.items] : inner.items, sections: inner.sections };
+        if (prune(section)) node.sections.push(section);
+      }
+      // A link leaf ({ href }) isn't a page of this site.
+    }
+    return node;
+  }
+
+  function fromContainer(container) {
+    if (Array.isArray(container)) return fromList(container);
+    if (!container || typeof container !== 'object') return { items: [], sections: [] };
+    if (Array.isArray(container.pages)) return fromList(container.pages);
+    for (const [key, labelKey] of CONTAINERS) {
+      if (!Array.isArray(container[key])) continue;
+      const sections = container[key]
+        .filter((child) => child && typeof child === 'object' && !('href' in child))
+        .map((child) => {
+          const label = labelKey === 'version' || labelKey === 'language' ? child.label ?? child[labelKey] : child[labelKey];
+          return { label: String(label), ...fromContainer(child) };
+        })
+        .filter(prune);
+      return { items: [], sections };
+    }
+    return { items: [], sections: [] };
+  }
+
+  const root = fromContainer(navigation);
+  for (const dropdown of (!Array.isArray(navigation) && navigation?.global?.dropdowns) || []) {
+    if ('href' in dropdown) continue;
+    const section = { label: String(dropdown.dropdown), ...fromContainer(dropdown) };
+    if (prune(section)) root.sections.push(section);
+  }
+  const rest = unlisted.map((id) => page(id)).filter(Boolean);
+  if (rest.length) root.sections.push({ label: 'Other pages', items: rest, sections: [] });
+  return root;
+}
+
+const pageLine = (p) => `- [${p.title}](${p.href})${p.description ? `: ${p.description}` : ''}`;
+const countPages = (s) => s.items.length + s.sections.reduce((n, c) => n + countPages(c), 0);
+const heading = (depth, label) => `${'#'.repeat(Math.min(depth, 6))} ${label}`;
+
+function slugify(label) {
+  return String(label).toLowerCase().replace(/[^a-z0-9]+/g, '-').replace(/^-+|-+$/g, '') || 'section';
+}
+
+/** The body of `node` as Markdown lines: its pages, then each sub-section -
+ *  under its own heading, or, when it's in `split`, as a link to its file. */
+function renderBody(node, depth, split) {
+  const lines = [];
+  if (node.items.length) lines.push(...node.items.map(pageLine), '');
+  for (const section of node.sections) {
+    lines.push(heading(depth, section.label), '');
+    const file = split.get(section);
+    if (file) {
+      const n = countPages(section);
+      lines.push(`- [${section.label}](${file.url}): ${n} page${n === 1 ? '' : 's'}, listed in their own file`, '');
+    } else {
+      lines.push(...renderBody(section, depth + 1, split));
+    }
+  }
+  return lines;
+}
+
+const render = (header, node, depth, split) => [...header, ...renderBody(node, depth, split)].join('\n').replace(/\n{3,}/g, '\n\n').trimEnd() + '\n';
+
+/** Every section inside `node` still rendered inline (not split, and not
+ *  inside a split one), with the characters it takes. */
+function inlineSections(node, depth, split, out = []) {
+  for (const section of node.sections) {
+    if (split.has(section)) continue;
+    out.push({ section, size: render([], { items: [], sections: [section] }, depth, split).length });
+    inlineSections(section, depth + 1, split, out);
+  }
+  return out;
+}
+
+/** A section whose own page list alone is too big becomes parts, each a
+ *  sub-section small enough for a file of its own. */
+function chunkItems(node, maxChars, label) {
+  const budget = Math.max(1000, maxChars - 2000);
+  const parts = [];
+  let current = [];
+  let size = 0;
+  for (const p of node.items) {
+    const n = pageLine(p).length + 1;
+    if (current.length && size + n > budget) {
+      parts.push(current);
+      current = [];
+      size = 0;
+    }
+    current.push(p);
+    size += n;
+  }
+  if (current.length) parts.push(current);
+  // One part is the list as it was - nothing smaller to split into (a single
+  // page line longer than the budget). Leave it, rather than loop.
+  if (parts.length < 2) return [];
+  const sections = parts.map((items, i) => ({ label: `${label} (part ${i + 1} of ${parts.length})`, items, sections: [] }));
+  node.sections = [...sections, ...node.sections];
+  node.items = [];
+  return sections;
+}
+
+/** llms.txt, and any child files it needs, as Map<path, text>: 'llms.txt',
+ *  then e.g. 'llms/core-platform.md'. `urlFor(path)` gives a file's URL. */
+export function renderLlmsFiles({ name, description, tree, maxChars, urlFor }) {
+  const files = new Map();
+  const taken = new Set(['llms.txt']);
+  const rootUrl = urlFor('llms.txt');
+
+  function emit(filePath, header, node, depth, trail) {
+    const split = new Map();
+    const splitOut = (section) => {
+      const dir = filePath === 'llms.txt' ? 'llms' : filePath.replace(/\.md$/, '');
+      let childPath = `${dir}/${slugify(section.label)}.md`;
+      for (let n = 2; taken.has(childPath); n++) childPath = `${dir}/${slugify(section.label)}-${n}.md`;
+      taken.add(childPath);
+      split.set(section, { path: childPath, url: urlFor(childPath) });
+    };
+    // A page list too long for this file on its own becomes parts - all of
+    // them separate files, so this one is a plain list of the parts.
+    if (render(header, { items: node.items, sections: [] }, depth, split).length > maxChars) {
+      chunkItems(node, maxChars, trail.length ? trail[trail.length - 1] : 'Pages').forEach(splitOut);
+    }
+    // Then the biggest sections move out, one at a time, until this fits.
+    while (render(header, node, depth, split).length > maxChars) {
+      const candidates = inlineSections(node, depth, split);
+      if (!candidates.length) break;
+      splitOut(candidates.reduce((a, b) => (b.size > a.size ? b : a)).section);
+    }
+    files.set(filePath, render(header, node, depth, split));
+    for (const [section, file] of split) {
+      const childTrail = [...trail, section.label];
+      const childHeader = [`# ${name}: ${childTrail.join(' > ')}`, '', `> Part of the ${name} docs index: ${rootUrl}`, ''];
+      emit(file.path, childHeader, section, 2, childTrail);
+    }
+  }
+
+  const header = [`# ${name}`, ''];
+  if (description) header.push(`> ${description}`, '');
+  emit('llms.txt', header, tree, 2, []);
+  return files;
+}