docusaurus-plugin-llms 0.1.0

package/README.md

@@ -0,0 +1,129 @@
+# 📜 docusaurus-plugin-llms
+
+A Docusaurus plugin for generating LLM-friendly documentation following the [llmtxt standard](https://llmtxt.org/).
+
+## Installation
+
+There are two ways to use this plugin:
+
+### 1. Direct Integration (Simplest Method)
+
+For quick integration, create a plugin file directly in your Docusaurus project:
+
+```bash
+mkdir -p src/plugins/llms
+```
+
+Then create a file at `src/plugins/llms/index.js` with the plugin code. Finally, add it to your `docusaurus.config.js`:
+
+```js
+module.exports = {
+  // ... your existing Docusaurus config
+  plugins: [
+    require('./src/plugins/llms'),
+    // ... your other plugins
+  ],
+};
+```
+
+### 2. As a Package (Not Yet Published)
+
+```bash
+npm install docusaurus-plugin-llms --save-dev
+```
+
+Then add to your Docusaurus configuration:
+
+```js
+module.exports = {
+  // ... your existing Docusaurus config
+  plugins: [
+    'docusaurus-plugin-llms',
+    // ... your other plugins
+  ],
+};
+```
+
+## Configuration Options
+
+You can configure the plugin by passing options:
+
+```js
+module.exports = {
+  // ... your existing Docusaurus config
+  plugins: [
+    [
+      'docusaurus-plugin-llms',
+      {
+        // Options here
+        generateLLMsTxt: true,
+        generateLLMsFullTxt: true,
+        docsDir: 'docs',
+        ignoreFiles: ['advanced/*', 'private/*'],
+        title: 'My Project Documentation',
+        description: 'Complete reference documentation for My Project',
+        includeBlog: true,
+      },
+    ],
+    // ... your other plugins
+  ],
+};
+```
+
+### Available Options
+
+| Option | Type | Default | Description |
+|--------|------|---------|-------------|
+| `generateLLMsTxt` | boolean | `true` | Whether to generate the links file |
+| `generateLLMsFullTxt` | boolean | `true` | Whether to generate the full content file |
+| `docsDir` | string | `'docs'` | Base directory for documentation files |
+| `ignoreFiles` | string[] | `[]` | Array of glob patterns for files to ignore |
+| `title` | string | Site title | Custom title to use in generated files |
+| `description` | string | Site tagline | Custom description to use in generated files |
+| `llmsTxtFilename` | string | `'llms.txt'` | Custom filename for the links file |
+| `llmsFullTxtFilename` | string | `'llms-full.txt'` | Custom filename for the full content file |
+| `includeBlog` | boolean | `false` | Whether to include blog content |
+
+## How It Works
+
+This plugin automatically generates the following files during the build process:
+
+- **llms.txt**: Contains links to all sections of your documentation
+- **llms-full.txt**: Contains all documentation content in a single file
+
+These files follow the [llmtxt standard](https://llmtxt.org/), making your documentation optimized for use with Large Language Models (LLMs).
+
+## Features
+
+- ⚡️ Easy integration with Docusaurus
+- ✅ Zero config required, works out of the box
+- ⚙️ Highly customizable with multiple options
+- 📝 Creates `llms.txt` with section links
+- 📖 Produces `llms-full.txt` with all content in one file
+- 🧹 Cleans HTML and normalizes content for optimal LLM consumption
+- 📊 Provides statistics about generated documentation
+- 📚 Option to include blog posts
+
+## Implementation Details
+
+The plugin:
+
+1. Scans your `docs` directory recursively for all Markdown files
+2. Optionally includes blog content
+3. Extracts metadata, titles, and content from each file
+4. Creates proper URL links to each document section
+5. Generates a table of contents in `llms.txt`
+6. Combines all documentation content in `llms-full.txt`
+7. Provides statistics about the generated documentation
+
+## Future Enhancements
+
+Planned features for future versions:
+
+- Advanced glob pattern matching for file filtering
+- Support for i18n content
+- Specific content tags for LLM-only sections
+
+## License
+
+MIT 

package/lib/index.d.ts

@@ -0,0 +1,43 @@
+/**
+ * @fileoverview Docusaurus plugin that generates LLM-friendly documentation following the llmtxt.org standard.
+ *
+ * This plugin creates two files:
+ * - llms.txt: Contains links to all sections of documentation
+ * - llms-full.txt: Contains all documentation content in a single file
+ *
+ * The plugin runs during the Docusaurus build process and scans all Markdown files in the docs directory.
+ */
+import type { LoadContext, Plugin } from '@docusaurus/types';
+/**
+ * Plugin options interface
+ */
+interface PluginOptions {
+    /** Whether to generate the llms.txt file (default: true) */
+    generateLLMsTxt?: boolean;
+    /** Whether to generate the llms-full.txt file (default: true) */
+    generateLLMsFullTxt?: boolean;
+    /** Base directory for documentation files (default: 'docs') */
+    docsDir?: string;
+    /** Array of glob patterns for files to ignore */
+    ignoreFiles?: string[];
+    /** Custom title to use in generated files (defaults to site title) */
+    title?: string;
+    /** Custom description to use in generated files (defaults to site tagline) */
+    description?: string;
+    /** Custom file name for the links file (default: 'llms.txt') */
+    llmsTxtFilename?: string;
+    /** Custom file name for the full content file (default: 'llms-full.txt') */
+    llmsFullTxtFilename?: string;
+    /** Whether to include blog content (default: false) */
+    includeBlog?: boolean;
+}
+/**
+ * A Docusaurus plugin to generate LLM-friendly documentation following
+ * the llmtxt.org standard
+ *
+ * @param context - Docusaurus context
+ * @param options - Plugin options
+ * @returns Plugin object
+ */
+export default function docusaurusPluginLLMs(context: LoadContext, options?: PluginOptions): Plugin<void>;
+export {};

package/lib/index.js

@@ -0,0 +1,330 @@
+"use strict";
+/**
+ * @fileoverview Docusaurus plugin that generates LLM-friendly documentation following the llmtxt.org standard.
+ *
+ * This plugin creates two files:
+ * - llms.txt: Contains links to all sections of documentation
+ * - llms-full.txt: Contains all documentation content in a single file
+ *
+ * The plugin runs during the Docusaurus build process and scans all Markdown files in the docs directory.
+ */
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.default = docusaurusPluginLLMs;
+const fs = __importStar(require("fs/promises"));
+const path = __importStar(require("path"));
+const gray_matter_1 = __importDefault(require("gray-matter"));
+const minimatch_1 = require("minimatch");
+/**
+ * Write content to a file
+ * @param filePath - Path to write the file to
+ * @param data - Content to write
+ */
+async function writeFile(filePath, data) {
+    return fs.writeFile(filePath, data, 'utf8');
+}
+/**
+ * Read content from a file
+ * @param filePath - Path of the file to read
+ * @returns Content of the file
+ */
+async function readFile(filePath) {
+    return fs.readFile(filePath, 'utf8');
+}
+/**
+ * Check if a file should be ignored based on glob patterns
+ * @param filePath - Path to the file
+ * @param baseDir - Base directory for relative paths
+ * @param ignorePatterns - Glob patterns for files to ignore
+ * @returns Whether the file should be ignored
+ */
+function shouldIgnoreFile(filePath, baseDir, ignorePatterns) {
+    if (ignorePatterns.length === 0) {
+        return false;
+    }
+    const relativePath = path.relative(baseDir, filePath);
+    return ignorePatterns.some(pattern => (0, minimatch_1.minimatch)(relativePath, pattern, { matchBase: true }));
+}
+/**
+ * Recursively reads all Markdown files in a directory
+ * @param dir - Directory to scan
+ * @param baseDir - Base directory for relative paths
+ * @param ignorePatterns - Glob patterns for files to ignore
+ * @returns Array of file paths
+ */
+async function readMarkdownFiles(dir, baseDir, ignorePatterns = []) {
+    const files = [];
+    const entries = await fs.readdir(dir, { withFileTypes: true });
+    for (const entry of entries) {
+        const fullPath = path.join(dir, entry.name);
+        if (shouldIgnoreFile(fullPath, baseDir, ignorePatterns)) {
+            continue;
+        }
+        if (entry.isDirectory()) {
+            const subDirFiles = await readMarkdownFiles(fullPath, baseDir, ignorePatterns);
+            files.push(...subDirFiles);
+        }
+        else if (entry.name.endsWith('.md') || entry.name.endsWith('.mdx')) {
+            files.push(fullPath);
+        }
+    }
+    return files;
+}
+/**
+ * Extract title from content or use the filename
+ * @param data - Frontmatter data
+ * @param content - Markdown content
+ * @param filePath - Path to the file
+ * @returns Extracted title
+ */
+function extractTitle(data, content, filePath) {
+    // First try frontmatter
+    if (data.title) {
+        return data.title;
+    }
+    // Then try first heading
+    const headingMatch = content.match(/^#\s+(.*)/m);
+    if (headingMatch) {
+        return headingMatch[1].trim();
+    }
+    // Finally use filename
+    return path.basename(filePath, path.extname(filePath))
+        .replace(/-/g, ' ')
+        .replace(/\b\w/g, c => c.toUpperCase());
+}
+/**
+ * Clean markdown content for LLM consumption
+ * @param content - Raw markdown content
+ * @returns Cleaned content
+ */
+function cleanMarkdownContent(content) {
+    // Remove HTML tags
+    let cleaned = content.replace(/<[^>]*>/g, '');
+    // Normalize whitespace
+    cleaned = cleaned.replace(/\r\n/g, '\n')
+        .replace(/\n{3,}/g, '\n\n')
+        .trim();
+    return cleaned;
+}
+/**
+ * Process a markdown file and extract its metadata and content
+ * @param filePath - Path to the markdown file
+ * @param baseDir - Base directory
+ * @param siteUrl - Base URL of the site
+ * @param pathPrefix - Path prefix for URLs (e.g., 'docs' or 'blog')
+ * @returns Processed file data
+ */
+async function processMarkdownFile(filePath, baseDir, siteUrl, pathPrefix = 'docs') {
+    const content = await readFile(filePath);
+    const { data, content: markdownContent } = (0, gray_matter_1.default)(content);
+    const relativePath = path.relative(baseDir, filePath);
+    // Convert to URL path format (replace backslashes with forward slashes on Windows)
+    const normalizedPath = relativePath.replace(/\\/g, '/');
+    // Convert .md extension to appropriate path
+    const linkPathBase = normalizedPath.replace(/\.mdx?$/, '');
+    // Handle index files specially
+    const linkPath = linkPathBase.endsWith('index')
+        ? linkPathBase.replace(/\/index$/, '')
+        : linkPathBase;
+    // Generate full URL
+    const fullUrl = new URL(`${pathPrefix}/${linkPath}`, siteUrl).toString();
+    // Extract title
+    const title = extractTitle(data, markdownContent, filePath);
+    // Get description from frontmatter or first paragraph
+    let description = data.description || '';
+    if (!description) {
+        const paragraphs = markdownContent.split('\n\n');
+        for (const para of paragraphs) {
+            if (para.trim() && !para.startsWith('#')) {
+                description = para.trim();
+                break;
+            }
+        }
+    }
+    // Clean and process content
+    const cleanedContent = cleanMarkdownContent(markdownContent);
+    return {
+        title,
+        path: normalizedPath,
+        url: fullUrl,
+        content: cleanedContent,
+        description: description || '',
+    };
+}
+/**
+ * A Docusaurus plugin to generate LLM-friendly documentation following
+ * the llmtxt.org standard
+ *
+ * @param context - Docusaurus context
+ * @param options - Plugin options
+ * @returns Plugin object
+ */
+function docusaurusPluginLLMs(context, options = {}) {
+    // Set default options
+    const { generateLLMsTxt = true, generateLLMsFullTxt = true, docsDir = 'docs', ignoreFiles = [], title, description, llmsTxtFilename = 'llms.txt', llmsFullTxtFilename = 'llms-full.txt', includeBlog = false, } = options;
+    const { siteDir, siteConfig, outDir, } = context;
+    return {
+        name: 'docusaurus-plugin-llms',
+        /**
+         * Generates LLM-friendly documentation files after the build is complete
+         */
+        async postBuild() {
+            console.log('Generating LLM-friendly documentation...');
+            // Custom title and description or fallback to site values
+            const docTitle = title || siteConfig.title;
+            const docDescription = description || siteConfig.tagline || '';
+            // Build the site URL with proper trailing slash
+            const siteUrl = siteConfig.url + (siteConfig.baseUrl.endsWith('/')
+                ? siteConfig.baseUrl.slice(0, -1)
+                : siteConfig.baseUrl || '');
+            // Initialize docs collection
+            const allDocs = [];
+            try {
+                // Process docs directory
+                const fullDocsDir = path.join(siteDir, docsDir);
+                try {
+                    await fs.access(fullDocsDir);
+                    // Collect all markdown files from docs directory
+                    const docFiles = await readMarkdownFiles(fullDocsDir, siteDir, ignoreFiles);
+                    if (docFiles.length > 0) {
+                        // Process each file
+                        for (const filePath of docFiles) {
+                            try {
+                                const docInfo = await processMarkdownFile(filePath, fullDocsDir, siteUrl, 'docs');
+                                allDocs.push(docInfo);
+                            }
+                            catch (err) {
+                                console.warn(`Error processing ${filePath}: ${err.message}`);
+                            }
+                        }
+                        console.log(`Processed ${docFiles.length} documentation files`);
+                    }
+                    else {
+                        console.warn('No markdown files found in docs directory.');
+                    }
+                }
+                catch (err) {
+                    console.warn(`Docs directory not found: ${fullDocsDir}`);
+                }
+                // Process blog if enabled
+                if (includeBlog) {
+                    const blogDir = path.join(siteDir, 'blog');
+                    try {
+                        await fs.access(blogDir);
+                        // Collect all markdown files from blog directory
+                        const blogFiles = await readMarkdownFiles(blogDir, siteDir, ignoreFiles);
+                        if (blogFiles.length > 0) {
+                            // Process each file
+                            for (const filePath of blogFiles) {
+                                try {
+                                    const docInfo = await processMarkdownFile(filePath, blogDir, siteUrl, 'blog');
+                                    allDocs.push(docInfo);
+                                }
+                                catch (err) {
+                                    console.warn(`Error processing ${filePath}: ${err.message}`);
+                                }
+                            }
+                            console.log(`Processed ${blogFiles.length} blog files`);
+                        }
+                        else {
+                            console.warn('No markdown files found in blog directory.');
+                        }
+                    }
+                    catch (err) {
+                        console.warn(`Blog directory not found: ${blogDir}`);
+                    }
+                }
+                // Skip further processing if no documents were found
+                if (allDocs.length === 0) {
+                    console.warn('No documents found to process.');
+                    return;
+                }
+                // Sort files to ensure consistent ordering
+                allDocs.sort((a, b) => a.title.localeCompare(b.title));
+                // Generate llms.txt
+                if (generateLLMsTxt) {
+                    const llmsTxtPath = path.join(outDir, llmsTxtFilename);
+                    const tocItems = allDocs.map(doc => {
+                        return `- [${doc.title}](${doc.url})${doc.description ? `: ${doc.description.split('\n')[0]}` : ''}`;
+                    });
+                    const llmsTxtContent = `# ${docTitle}
+
+> ${docDescription}
+
+This file contains links to all documentation sections following the llmtxt.org standard.
+
+## Table of Contents
+
+${tocItems.join('\n')}
+`;
+                    await writeFile(llmsTxtPath, llmsTxtContent);
+                    console.log(`Generated ${llmsTxtFilename}: ${llmsTxtPath}`);
+                }
+                // Generate llms-full.txt with all content
+                if (generateLLMsFullTxt) {
+                    const llmsFullTxtPath = path.join(outDir, llmsFullTxtFilename);
+                    const fullContentSections = allDocs.map(doc => {
+                        return `## ${doc.title}
+
+${doc.content}`;
+                    });
+                    const llmsFullTxtContent = `# ${docTitle}
+
+> ${docDescription}
+
+This file contains all documentation content in a single document following the llmtxt.org standard.
+
+${fullContentSections.join('\n\n---\n\n')}
+`;
+                    await writeFile(llmsFullTxtPath, llmsFullTxtContent);
+                    console.log(`Generated ${llmsFullTxtFilename}: ${llmsFullTxtPath}`);
+                }
+                // Output statistics
+                const stats = {
+                    totalDocuments: allDocs.length,
+                    totalBytes: allDocs.reduce((sum, doc) => sum + doc.content.length, 0),
+                    approxTokens: Math.round(allDocs.reduce((sum, doc) => sum + doc.content.length, 0) / 4), // Rough token estimate
+                };
+                console.log(`Stats: ${stats.totalDocuments} documents, ${Math.round(stats.totalBytes / 1024)}KB, ~${stats.approxTokens} tokens`);
+            }
+            catch (err) {
+                console.error('Error generating LLM documentation:', err);
+            }
+        },
+    };
+}

package/package.json

@@ -0,0 +1,52 @@
+{
+  "name": "docusaurus-plugin-llms",
+  "version": "0.1.0",
+  "description": "Docusaurus plugin for generating LLM-friendly documentation following the llmtxt.org standard",
+  "main": "lib/index.js",
+  "scripts": {
+    "build": "tsc",
+    "watch": "tsc --watch",
+    "cleanup": "node cleanup.js",
+    "prepublishOnly": "npm run build && npm run cleanup",
+    "test": "echo \"No tests specified\""
+  },
+  "files": [
+    "lib",
+    "src"
+  ],
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/rachfop/docusaurus-plugin-llms.git"
+  },
+  "bugs": {
+    "url": "https://github.com/rachfop/docusaurus-plugin-llms/issues"
+  },
+  "homepage": "https://github.com/rachfop/docusaurus-plugin-llms#readme",
+  "keywords": [
+    "docusaurus",
+    "docusaurus-plugin",
+    "documentation",
+    "llm",
+    "llms",
+    "llmtxt"
+  ],
+  "author": "Patrick Rachford",
+  "email": "prachford@icloud.com",
+  "license": "MIT",
+  "dependencies": {
+    "gray-matter": "^4.0.3",
+    "minimatch": "^9.0.3"
+  },
+  "peerDependencies": {
+    "@docusaurus/core": "^3.0.0"
+  },
+  "devDependencies": {
+    "@docusaurus/types": "^3.0.0",
+    "@types/minimatch": "^5.1.2",
+    "@types/node": "^20.6.0",
+    "typescript": "^5.2.2"
+  },
+  "engines": {
+    "node": ">=18.0"
+  }
+}

package/src/index.ts

@@ -0,0 +1,415 @@
+/**
+ * @fileoverview Docusaurus plugin that generates LLM-friendly documentation following the llmtxt.org standard.
+ * 
+ * This plugin creates two files:
+ * - llms.txt: Contains links to all sections of documentation
+ * - llms-full.txt: Contains all documentation content in a single file
+ * 
+ * The plugin runs during the Docusaurus build process and scans all Markdown files in the docs directory.
+ */
+
+import * as fs from 'fs/promises';
+import * as path from 'path';
+import matter from 'gray-matter';
+import { minimatch } from 'minimatch';
+import type { LoadContext, Plugin } from '@docusaurus/types';
+
+/**
+ * Interface for processed document information
+ */
+interface DocInfo {
+  title: string;
+  path: string;
+  url: string;
+  content: string;
+  description: string;
+}
+
+/**
+ * Plugin options interface
+ */
+interface PluginOptions {
+  /** Whether to generate the llms.txt file (default: true) */
+  generateLLMsTxt?: boolean;
+  
+  /** Whether to generate the llms-full.txt file (default: true) */
+  generateLLMsFullTxt?: boolean;
+  
+  /** Base directory for documentation files (default: 'docs') */
+  docsDir?: string;
+  
+  /** Array of glob patterns for files to ignore */
+  ignoreFiles?: string[];
+  
+  /** Custom title to use in generated files (defaults to site title) */
+  title?: string;
+  
+  /** Custom description to use in generated files (defaults to site tagline) */
+  description?: string;
+  
+  /** Custom file name for the links file (default: 'llms.txt') */
+  llmsTxtFilename?: string;
+  
+  /** Custom file name for the full content file (default: 'llms-full.txt') */
+  llmsFullTxtFilename?: string;
+  
+  /** Whether to include blog content (default: false) */
+  includeBlog?: boolean;
+}
+
+/**
+ * Write content to a file
+ * @param filePath - Path to write the file to
+ * @param data - Content to write
+ */
+async function writeFile(filePath: string, data: string): Promise<void> {
+  return fs.writeFile(filePath, data, 'utf8');
+}
+
+/**
+ * Read content from a file
+ * @param filePath - Path of the file to read
+ * @returns Content of the file
+ */
+async function readFile(filePath: string): Promise<string> {
+  return fs.readFile(filePath, 'utf8');
+}
+
+/**
+ * Check if a file should be ignored based on glob patterns
+ * @param filePath - Path to the file
+ * @param baseDir - Base directory for relative paths
+ * @param ignorePatterns - Glob patterns for files to ignore
+ * @returns Whether the file should be ignored
+ */
+function shouldIgnoreFile(filePath: string, baseDir: string, ignorePatterns: string[]): boolean {
+  if (ignorePatterns.length === 0) {
+    return false;
+  }
+  
+  const relativePath = path.relative(baseDir, filePath);
+  
+  return ignorePatterns.some(pattern => 
+    minimatch(relativePath, pattern, { matchBase: true })
+  );
+}
+
+/**
+ * Recursively reads all Markdown files in a directory
+ * @param dir - Directory to scan
+ * @param baseDir - Base directory for relative paths
+ * @param ignorePatterns - Glob patterns for files to ignore
+ * @returns Array of file paths
+ */
+async function readMarkdownFiles(dir: string, baseDir: string, ignorePatterns: string[] = []): Promise<string[]> {
+  const files: string[] = [];
+  const entries = await fs.readdir(dir, { withFileTypes: true });
+
+  for (const entry of entries) {
+    const fullPath = path.join(dir, entry.name);
+    
+    if (shouldIgnoreFile(fullPath, baseDir, ignorePatterns)) {
+      continue;
+    }
+    
+    if (entry.isDirectory()) {
+      const subDirFiles = await readMarkdownFiles(fullPath, baseDir, ignorePatterns);
+      files.push(...subDirFiles);
+    } else if (entry.name.endsWith('.md') || entry.name.endsWith('.mdx')) {
+      files.push(fullPath);
+    }
+  }
+
+  return files;
+}
+
+/**
+ * Extract title from content or use the filename
+ * @param data - Frontmatter data
+ * @param content - Markdown content
+ * @param filePath - Path to the file
+ * @returns Extracted title
+ */
+function extractTitle(data: any, content: string, filePath: string): string {
+  // First try frontmatter
+  if (data.title) {
+    return data.title;
+  }
+  
+  // Then try first heading
+  const headingMatch = content.match(/^#\s+(.*)/m);
+  if (headingMatch) {
+    return headingMatch[1].trim();
+  }
+  
+  // Finally use filename
+  return path.basename(filePath, path.extname(filePath))
+    .replace(/-/g, ' ')
+    .replace(/\b\w/g, c => c.toUpperCase());
+}
+
+/**
+ * Clean markdown content for LLM consumption
+ * @param content - Raw markdown content
+ * @returns Cleaned content
+ */
+function cleanMarkdownContent(content: string): string {
+  // Remove HTML tags
+  let cleaned = content.replace(/<[^>]*>/g, '');
+  
+  // Normalize whitespace
+  cleaned = cleaned.replace(/\r\n/g, '\n')
+    .replace(/\n{3,}/g, '\n\n')
+    .trim();
+    
+  return cleaned;
+}
+
+/**
+ * Process a markdown file and extract its metadata and content
+ * @param filePath - Path to the markdown file
+ * @param baseDir - Base directory
+ * @param siteUrl - Base URL of the site
+ * @param pathPrefix - Path prefix for URLs (e.g., 'docs' or 'blog')
+ * @returns Processed file data
+ */
+async function processMarkdownFile(
+  filePath: string, 
+  baseDir: string, 
+  siteUrl: string,
+  pathPrefix: string = 'docs'
+): Promise<DocInfo> {
+  const content = await readFile(filePath);
+  const { data, content: markdownContent } = matter(content);
+  
+  const relativePath = path.relative(baseDir, filePath);
+  // Convert to URL path format (replace backslashes with forward slashes on Windows)
+  const normalizedPath = relativePath.replace(/\\/g, '/');
+  
+  // Convert .md extension to appropriate path
+  const linkPathBase = normalizedPath.replace(/\.mdx?$/, '');
+  
+  // Handle index files specially
+  const linkPath = linkPathBase.endsWith('index') 
+    ? linkPathBase.replace(/\/index$/, '') 
+    : linkPathBase;
+  
+  // Generate full URL
+  const fullUrl = new URL(`${pathPrefix}/${linkPath}`, siteUrl).toString();
+  
+  // Extract title
+  const title = extractTitle(data, markdownContent, filePath);
+  
+  // Get description from frontmatter or first paragraph
+  let description = data.description || '';
+  if (!description) {
+    const paragraphs = markdownContent.split('\n\n');
+    for (const para of paragraphs) {
+      if (para.trim() && !para.startsWith('#')) {
+        description = para.trim();
+        break;
+      }
+    }
+  }
+  
+  // Clean and process content
+  const cleanedContent = cleanMarkdownContent(markdownContent);
+  
+  return {
+    title,
+    path: normalizedPath,
+    url: fullUrl,
+    content: cleanedContent,
+    description: description || '',
+  };
+}
+
+/**
+ * A Docusaurus plugin to generate LLM-friendly documentation following
+ * the llmtxt.org standard
+ * 
+ * @param context - Docusaurus context
+ * @param options - Plugin options
+ * @returns Plugin object
+ */
+export default function docusaurusPluginLLMs(
+  context: LoadContext,
+  options: PluginOptions = {}
+): Plugin<void> {
+  // Set default options
+  const {
+    generateLLMsTxt = true,
+    generateLLMsFullTxt = true,
+    docsDir = 'docs',
+    ignoreFiles = [],
+    title,
+    description,
+    llmsTxtFilename = 'llms.txt',
+    llmsFullTxtFilename = 'llms-full.txt',
+    includeBlog = false,
+  } = options;
+
+  const {
+    siteDir,
+    siteConfig,
+    outDir,
+  } = context;
+
+  return {
+    name: 'docusaurus-plugin-llms',
+
+    /**
+     * Generates LLM-friendly documentation files after the build is complete
+     */
+    async postBuild(): Promise<void> {
+      console.log('Generating LLM-friendly documentation...');
+
+      // Custom title and description or fallback to site values
+      const docTitle = title || siteConfig.title;
+      const docDescription = description || siteConfig.tagline || '';
+      
+      // Build the site URL with proper trailing slash
+      const siteUrl = siteConfig.url + (
+        siteConfig.baseUrl.endsWith('/') 
+          ? siteConfig.baseUrl.slice(0, -1) 
+          : siteConfig.baseUrl || ''
+      );
+      
+      // Initialize docs collection
+      const allDocs: DocInfo[] = [];
+
+      try {
+        // Process docs directory
+        const fullDocsDir = path.join(siteDir, docsDir);
+        
+        try {
+          await fs.access(fullDocsDir);
+          
+          // Collect all markdown files from docs directory
+          const docFiles = await readMarkdownFiles(fullDocsDir, siteDir, ignoreFiles);
+          
+          if (docFiles.length > 0) {
+            // Process each file
+            for (const filePath of docFiles) {
+              try {
+                const docInfo = await processMarkdownFile(
+                  filePath, 
+                  fullDocsDir, 
+                  siteUrl,
+                  'docs'
+                );
+                allDocs.push(docInfo);
+              } catch (err: any) {
+                console.warn(`Error processing ${filePath}: ${err.message}`);
+              }
+            }
+            console.log(`Processed ${docFiles.length} documentation files`);
+          } else {
+            console.warn('No markdown files found in docs directory.');
+          }
+        } catch (err) {
+          console.warn(`Docs directory not found: ${fullDocsDir}`);
+        }
+        
+        // Process blog if enabled
+        if (includeBlog) {
+          const blogDir = path.join(siteDir, 'blog');
+          
+          try {
+            await fs.access(blogDir);
+            
+            // Collect all markdown files from blog directory
+            const blogFiles = await readMarkdownFiles(blogDir, siteDir, ignoreFiles);
+            
+            if (blogFiles.length > 0) {
+              // Process each file
+              for (const filePath of blogFiles) {
+                try {
+                  const docInfo = await processMarkdownFile(
+                    filePath, 
+                    blogDir, 
+                    siteUrl,
+                    'blog'
+                  );
+                  allDocs.push(docInfo);
+                } catch (err: any) {
+                  console.warn(`Error processing ${filePath}: ${err.message}`);
+                }
+              }
+              console.log(`Processed ${blogFiles.length} blog files`);
+            } else {
+              console.warn('No markdown files found in blog directory.');
+            }
+          } catch (err) {
+            console.warn(`Blog directory not found: ${blogDir}`);
+          }
+        }
+        
+        // Skip further processing if no documents were found
+        if (allDocs.length === 0) {
+          console.warn('No documents found to process.');
+          return;
+        }
+        
+        // Sort files to ensure consistent ordering
+        allDocs.sort((a, b) => a.title.localeCompare(b.title));
+
+        // Generate llms.txt
+        if (generateLLMsTxt) {
+          const llmsTxtPath = path.join(outDir, llmsTxtFilename);
+          const tocItems = allDocs.map(doc => {
+            return `- [${doc.title}](${doc.url})${doc.description ? `: ${doc.description.split('\n')[0]}` : ''}`;
+          });
+
+          const llmsTxtContent = `# ${docTitle}
+
+> ${docDescription}
+
+This file contains links to all documentation sections following the llmtxt.org standard.
+
+## Table of Contents
+
+${tocItems.join('\n')}
+`;
+
+          await writeFile(llmsTxtPath, llmsTxtContent);
+          console.log(`Generated ${llmsTxtFilename}: ${llmsTxtPath}`);
+        }
+
+        // Generate llms-full.txt with all content
+        if (generateLLMsFullTxt) {
+          const llmsFullTxtPath = path.join(outDir, llmsFullTxtFilename);
+          
+          const fullContentSections = allDocs.map(doc => {
+            return `## ${doc.title}
+
+${doc.content}`;
+          });
+
+          const llmsFullTxtContent = `# ${docTitle}
+
+> ${docDescription}
+
+This file contains all documentation content in a single document following the llmtxt.org standard.
+
+${fullContentSections.join('\n\n---\n\n')}
+`;
+
+          await writeFile(llmsFullTxtPath, llmsFullTxtContent);
+          console.log(`Generated ${llmsFullTxtFilename}: ${llmsFullTxtPath}`);
+        }
+        
+        // Output statistics
+        const stats = {
+          totalDocuments: allDocs.length,
+          totalBytes: allDocs.reduce((sum, doc) => sum + doc.content.length, 0),
+          approxTokens: Math.round(allDocs.reduce((sum, doc) => sum + doc.content.length, 0) / 4), // Rough token estimate
+        };
+        
+        console.log(`Stats: ${stats.totalDocuments} documents, ${Math.round(stats.totalBytes / 1024)}KB, ~${stats.approxTokens} tokens`);
+      } catch (err: any) {
+        console.error('Error generating LLM documentation:', err);
+      }
+    },
+  };
+}