botdocs 0.5.0 → 0.6.0

package/README.md

@@ -15,7 +15,8 @@ Convert markdown documentation into beautiful static sites with AI-powered seman
 - **No Backend** - Everything runs in the browser
 - **Fast** - Syntax highlighting with Shiki
 - **Live Preview** - `--watch` rebuilds on save and serves the site locally
-- **SEO** - Optional Open Graph/Twitter tags and `sitemap.xml` via `baseUrl`
+- **SEO** - Optional Open Graph/Twitter tags, `sitemap.xml` and `robots.txt` via `baseUrl`
+- **Agent Friendly** - Raw `.md` source published next to every page, plus an `llms.txt` index
 
 ## Installation
 
@@ -105,7 +106,7 @@ Create `botdocs.config.json` in your docs directory:
 | `theme` | string | `"classic"` | Theme to use (classic, material, minimal, slate, modern) |
 | `customCss` | string | none | Path to a CSS file, resolved relative to the config file's directory. Appended after theme CSS in `bundle.css`, so same-specificity selectors override the theme without `!important` |
 | `attribution` | boolean | `true` | Show "Built with Botdocs" footer link |
-| `baseUrl` | string | none | Canonical URL where the site is hosted. When set, pages get `rel=canonical` and Open Graph/Twitter card tags, and a `sitemap.xml` is generated |
+| `baseUrl` | string | none | Canonical URL where the site is hosted. When set, pages get `rel=canonical` and Open Graph/Twitter card tags, a `sitemap.xml` and `robots.txt` are generated, and `llms.txt` links become absolute |
 | `chat.enabled` | boolean | `true` | Enable AI chatbot |
 | `chat.welcomeMessage` | string | `"Ask me anything about the docs!"` | Chatbot welcome message |
 | `build.chunkSize` | number | `500` | Text chunk size for embeddings |

package/dist/src/builder/site-generator.d.ts

@@ -10,6 +10,8 @@ export declare class SiteGenerator {
      * Generate the complete site
      */
     generate(inputDir: string, outputDir: string, config: BotdocsConfig): Promise<ProcessedDocument[]>;
+    private writeLlmsTxt;
+    private writeRobotsTxt;
     private writeSitemap;
     /**
      * Build navigation structure from documents, grouping by top-level

package/dist/src/builder/site-generator.js

@@ -109,6 +109,7 @@ export class SiteGenerator {
             const outputPath = join(outputDir, doc.relativePath.replace(/\.md$/, '.html'));
             mkdirSync(dirname(outputPath), { recursive: true });
             writeFileSync(outputPath, html, 'utf-8');
+            cpSync(doc.filePath, join(outputDir, doc.relativePath));
         }
         // Copy index.html if README.md exists
         const readmeDoc = this.documents.find((d) => d.relativePath === 'README.md' || d.relativePath === 'index.md');
@@ -120,11 +121,34 @@ export class SiteGenerator {
             }
         }
         console.log(`Generated ${this.documents.length} HTML pages`);
+        this.writeLlmsTxt(outputDir, config);
         if (config.baseUrl) {
             this.writeSitemap(outputDir, config.baseUrl);
+            this.writeRobotsTxt(outputDir, config.baseUrl);
         }
         return this.documents;
     }
+    writeLlmsTxt(outputDir, config) {
+        const oneLine = (value) => stripHtml(value).replace(/\s+/g, ' ').trim();
+        const isRootIndex = (doc) => doc.relativePath === 'README.md' || doc.relativePath === 'index.md';
+        const pages = [...this.documents.filter(isRootIndex), ...this.documents.filter((doc) => !isRootIndex(doc))]
+            .map((doc) => {
+            const title = oneLine(doc.metadata.title || basename(doc.relativePath, '.md'));
+            const url = config.baseUrl ? absoluteUrl(config.baseUrl, doc.relativePath) : doc.relativePath;
+            const description = oneLine(doc.metadata.description || '');
+            return `- [${title}](${url})${description ? `: ${description}` : ''}`;
+        })
+            .join('\n');
+        const summary = oneLine(config.description || '');
+        const llms = `# ${oneLine(config.title || 'Documentation')}\n\n${summary ? `> ${summary}\n\n` : ''}## Docs\n\n${pages}\n`;
+        writeFileSync(join(outputDir, 'llms.txt'), llms, 'utf-8');
+        console.log('Generated llms.txt');
+    }
+    writeRobotsTxt(outputDir, baseUrl) {
+        const robots = `User-agent: *\nAllow: /\n\nSitemap: ${absoluteUrl(baseUrl, 'sitemap.xml')}\n`;
+        writeFileSync(join(outputDir, 'robots.txt'), robots, 'utf-8');
+        console.log('Generated robots.txt');
+    }
     writeSitemap(outputDir, baseUrl) {
         const urls = this.documents
             .map((doc) => {

package/man/botdocs.1

@@ -106,8 +106,18 @@ When set, each page emits
 .B rel=canonical
 and Open Graph/Twitter card metadata, and a
 .B sitemap.xml
-listing every generated page is written to the output directory.
-When unset, these tags are omitted and no sitemap is generated.
+listing every generated page and a
+.B robots.txt
+pointing at it are written to the output directory, and
+.B llms.txt
+links become absolute.
+When unset, these tags are omitted and no sitemap or robots.txt is generated.
+.PP
+Every build copies each page's markdown source next to its HTML and writes an
+.B llms.txt
+index linking to those
+.B .md
+files, so agents can read the docs without parsing HTML.
 .PP
 .B build.minChunkSize
 folds chunks smaller than this many estimated tokens into a neighboring

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "botdocs",
-  "version": "0.5.0",
+  "version": "0.6.0",
   "description": "CLI tool that converts markdown documentation into a static website with an optional client-side AI chatbot",
   "author": "usr-wwelsh <https://wwel.sh>",
   "license": "MIT",