@writedocs/generator 0.7.3 → 0.8.0

package/src/pages/llms.txt.ts

@@ -1,9 +1,7 @@
-import { getCollection, type CollectionEntry } from 'astro:content';
 import fs from 'node:fs';
 import path from 'node:path';
 import type { APIRoute } from 'astro';
-import { loadDocsConfig, normalizeEntryId, findAllPages, resolveSiteUrl } from '../lib/config';
-import { writedocsTempDir } from '../lib/writedocs-temp-dir.js';
+import { llmsIndexFiles } from '../lib/llms-index';
 
 // A lightweight, LLM-facing index of every page on the site - the
 // llms.txt convention (see https://llmstxt.org), also adopted by Mintlify
@@ -13,136 +11,30 @@ import { writedocsTempDir } from '../lib/writedocs-temp-dir.js';
 // of the pair; llms-full.txt (llms-full.txt.ts, right next to this file)
 // is the "everything, concatenated" half.
 //
-// Fixed, non-dynamic route (no getStaticPaths/params needed) - Astro
-// prerenders this once at the literal path /llms.txt, same as any other
-// static-output route (this project's astro.config.mjs sets
-// `output: 'static'`), and re-evaluates it per request in `writedocs dev`
-// - see [...slug].md.ts for the same fixed-route-vs-catchall distinction
-// applied to per-page raw Markdown.
+// Pages are listed under headings that follow the navigation. A site too
+// big for one file keeps this as a directory whose largest sections link to
+// child files - llms/[...path].md.ts serves those - so no page is left out;
+// see lib/llms.js.
 //
-// Unconditional - unlike the per-page .md routes in [...slug].md.ts,
-// this doesn't depend on writedocs.json's `contextMenu` field. There's no UI
+// Fixed, non-dynamic route - Astro prerenders this once at the literal path
+// /llms.txt (this project's astro.config.mjs sets `output: 'static'`), and
+// re-evaluates it per request in `writedocs dev`.
+//
+// Unconditional - unlike the per-page .md routes in [...slug].md.ts, this
+// doesn't depend on writedocs.json's `contextMenu` field. There's no UI
 // footprint to gate (it's an extra static file, not a visible menu), so
 // it's always generated. `contextMenu` only affects which URL each entry
-// links to below (a real fetchable .md URL if the per-page raw-Markdown
-// routes exist, the ordinary HTML page URL otherwise).
-type DocsEntry = CollectionEntry<'pages'> | CollectionEntry<'generatedDocs'>;
-
-// Mirrors Mintlify's own llms.txt behavior: truncate a page's frontmatter
-// `description` at the first line break (a multi-paragraph description
-// would blow out a one-line list entry) and at 300 characters (an
-// arbitrary but reasonable cap - keeps every entry scannable regardless
-// of how verbose an individual page's description happens to be).
-const DESCRIPTION_MAX_CHARS = 300;
-function truncateDescription(description: string | undefined): string | undefined {
-  if (!description) return undefined;
-  const firstLine = description.split('\n')[0].trim();
-  if (!firstLine) return undefined;
-  return firstLine.length > DESCRIPTION_MAX_CHARS
-    ? firstLine.slice(0, DESCRIPTION_MAX_CHARS).trimEnd() + '…'
-    : firstLine;
-}
-
-// Same cap Mintlify documents for its own auto-generated llms.txt - this
-// file is meant to stay a lightweight index (one line per page), not
-// balloon into something llms-full.txt-sized. Realistically unlikely to
-// matter for most Writedocs sites (a site would need on the order of a
-// thousand-plus pages with full-length descriptions to hit this), but
-// cheap to guard against regardless.
-const MAX_CHARS = 100_000;
-
+// links to (a real fetchable .md URL if the per-page raw-Markdown routes
+// exist, the ordinary HTML page URL otherwise).
 export const GET: APIRoute = async () => {
-  const contentDir = process.env.WRITEDOCS_CONTENT_DIR || process.cwd();
-
-  // A hand-authored llms.txt at the project root (next to writedocs.json)
-  // always wins outright over the generated one below - same override
-  // convention Mintlify documents for its own llms.txt. Lets a site
-  // author hand-curate this file (different ordering, editorial
-  // descriptions, extra sections) without losing the ability to fall
-  // back to the auto-generated version by just deleting the override.
-  const customPath = path.join(contentDir, 'llms.txt');
-  if (fs.existsSync(customPath)) {
-    return new Response(fs.readFileSync(customPath, 'utf-8'), {
-      headers: { 'Content-Type': 'text/plain; charset=utf-8' },
-    });
-  }
-
-  const config = loadDocsConfig(contentDir);
-  const siteUrl = resolveSiteUrl(config); // absolute origin if `domain` is set, else null
-
-  const hasGeneratedDocs = fs.existsSync(path.join(writedocsTempDir(contentDir), 'generated-docs'));
-  const hasPages = findAllPages(contentDir).length > 0;
-  const [pagesEntries, generatedDocsEntries] = await Promise.all([
-    hasPages ? getCollection('pages') : Promise.resolve([]),
-    hasGeneratedDocs ? getCollection('generatedDocs') : Promise.resolve([]),
-  ]);
-  const entries: DocsEntry[] = [...pagesEntries, ...generatedDocsEntries];
-
-  const items = entries
-    // A page that opted out of search-engine indexing (frontmatter
-    // `seo.noindex: true`, already excluded from sitemap.xml - see
-    // astro.config.mjs's collectNoindexIds()) is excluded here for the
-    // same reason: llms.txt exists to help external tools discover
-    // pages, exactly what `noindex` asked not to happen.
-    .filter((entry) => !entry.data.seo?.noindex)
-    // A frontmatter `url` page (Mintlify's external link) has no content -
-    // its own URL only redirects to the link.
-    .filter((entry) => !entry.data.url)
-    .map((entry: DocsEntry) => {
-      const slug = normalizeEntryId(entry.id);
-      // Same URL Astro's own router resolves this entry to - see
-      // hrefForSlug() in [...slug].astro for the HTML form, and
-      // [...slug].md.ts for the .md form (no trailing slash, "index.md"
-      // for the home page rather than the HTML convention's bare "/").
-      // An OpenAPI operation page has no .md form ([...slug].md.ts leaves
-      // it out - it renders from the spec, not prose), so it keeps its HTML
-      // URL even when the .md routes exist.
-      const hasMarkdownRoute = config.contextMenu && !entry.data.openapi;
-      const path_ = hasMarkdownRoute ? `/${slug}.md` : slug === 'index' ? '/' : `/${slug}/`;
-      const href = (siteUrl ?? '') + path_;
-      let description = truncateDescription(entry.data.description);
-      // Mirrors Mintlify's own behavior: an OpenAPI operation page's
-      // description gets its "METHOD /path" appended, since the page
-      // itself renders almost entirely from the spec at request time
-      // (see ApiPlayground.astro) rather than from frontmatter prose.
-      if (entry.data.openapi) {
-        description = description ? `${description} (${entry.data.openapi})` : entry.data.openapi;
-      }
-      return { title: entry.data.title, href, description, slug };
-    })
-    .sort((a, b) => a.slug.localeCompare(b.slug));
-
-  const lines: string[] = [`# ${config.name}`, ''];
-  if (config.description) lines.push(`> ${config.description}`, '');
-  lines.push('## Docs', '');
-  for (const item of items) {
-    const entryLine = `- [${item.title}](${item.href})${item.description ? `: ${item.description}` : ''}`;
-    lines.push(entryLine);
-  }
-
-  let content = lines.join('\n') + '\n';
-  if (content.length > MAX_CHARS) {
-    // Truncate at the last full line inside the budget, then note how
-    // many entries got cut - rather than silently producing a
-    // mid-sentence-cutoff file, or (worse) one JSON/Markdown-breaking
-    // half-written link.
-    const truncatedLines = lines.slice(0, 4 + (config.description ? 2 : 0)); // "# name" / "" / ["> desc" / ""] / "## Docs" / ""
-    let runningLength = truncatedLines.join('\n').length + 1;
-    let omitted = 0;
-    for (const item of items) {
-      const entryLine = `- [${item.title}](${item.href})${item.description ? `: ${item.description}` : ''}`;
-      if (runningLength + entryLine.length + 1 > MAX_CHARS - 200) {
-        omitted++;
-        continue;
-      }
-      truncatedLines.push(entryLine);
-      runningLength += entryLine.length + 1;
-    }
-    truncatedLines.push('', `_Truncated — ${omitted} more page(s) omitted. See llms-full.txt or the site's own navigation for the complete list._`);
-    content = truncatedLines.join('\n') + '\n';
-  }
-
-  return new Response(content, {
+  const files = await llmsIndexFiles();
+  // null: the project has its own llms.txt at its root, next to
+  // writedocs.json - it replaces the generated one (and its child files)
+  // outright, and a site can fall back by just deleting it.
+  const text = files
+    ? files.get('llms.txt')
+    : fs.readFileSync(path.join(process.env.WRITEDOCS_CONTENT_DIR || process.cwd(), 'llms.txt'), 'utf-8');
+  return new Response(text, {
     headers: { 'Content-Type': 'text/plain; charset=utf-8' },
   });
 };

package/writedocs.schema.json

@@ -772,6 +772,12 @@
       "description": "Adds a \"Copy page\" menu to pages - copy as Markdown, or open the page in an AI assistant - and a Markdown copy of each page at its address + \".md\".",
       "markdownDescription": "Adds a \"Copy page\" menu to pages - copy as Markdown, or open the page in an AI assistant - and a Markdown copy of each page at its address + \".md\"."
     },
+    "mcp": {
+      "default": true,
+      "type": "boolean",
+      "description": "An MCP server at /mcp, so AI tools can search and read the docs. The build adds its index and a Cloudflare-ready _worker.js to dist/. Default true; false leaves them out.",
+      "markdownDescription": "An MCP server at /mcp, so AI tools can search and read the docs. The build adds its index and a Cloudflare-ready _worker.js to dist/. Default true; false leaves them out."
+    },
     "redirects": {
       "default": [],
       "type": "array",