docusaurus-plugin-llms 0.1.4 → 0.2.0

package/README.md

@@ -14,6 +14,8 @@ A Docusaurus plugin for generating LLM-friendly documentation following the [llm
 - 📚 Option to include blog posts
 - 🧩 Custom LLM files for specific documentation sections
 - 🧹 Cleans HTML and normalizes content for optimal LLM consumption
+- 🚫 Optional import statement removal for cleaner MDX content
+- 🔄 Optional duplicate heading removal for concise output
 - 📊 Provides statistics about generated documentation
 
 ## Table of Contents
@@ -24,6 +26,8 @@ A Docusaurus plugin for generating LLM-friendly documentation following the [llm
 - [Path Transformation Examples](#path-transformation-examples)
 - [Document Ordering Examples](#document-ordering-examples)
 - [Custom LLM Files](#custom-llm-files)
+- [Content Cleaning Options](#content-cleaning-options)
+- [Best Practices](#best-practices)
 - [How It Works](#how-it-works)
 - [Implementation Details](#implementation-details)
 - [Testing](#testing)
@@ -67,6 +71,11 @@ module.exports = {
         title: 'My Project Documentation',
         description: 'Complete reference documentation for My Project',
         includeBlog: true,
+        // Content cleaning options
+        excludeImports: true,
+        removeDuplicateHeadings: true,
+        // Generate individual markdown files following llmstxt.org specification
+        generateMarkdownFiles: true,
         // Control documentation order
         includeOrder: [
           'getting-started/*',
@@ -111,6 +120,7 @@ module.exports = {
 |----------------------------------|----------|-------------------|---------------------------------------------------------------|
 | `description`                    | string   | Site tagline      | Custom description to use in generated files                  |
 | `docsDir`                        | string   | `'docs'`          | Base directory for documentation files                        |
+| `excludeImports`                 | boolean  | `false`           | Remove import statements from generated content                |
 | `generateLLMsFullTxt`            | boolean  | `true`            | Whether to generate the full content file                     |
 | `generateLLMsTxt`                | boolean  | `true`            | Whether to generate the links file                            |
 | `ignoreFiles`                    | string[] | `[]`              | Array of glob patterns for files to ignore                    |
@@ -122,9 +132,81 @@ module.exports = {
 | `pathTransformation.addPaths`    | string[] | `[]`              | Path segments to add when constructing URLs                   |
 | `pathTransformation.ignorePaths` | string[] | `[]`              | Path segments to ignore when constructing URLs                |
 | `pathTransformation`             | object   | `undefined`       | Path transformation options for URL construction              |
+| `removeDuplicateHeadings`        | boolean  | `false`           | Remove redundant content that duplicates heading text         |
 | `title`                          | string   | Site title        | Custom title to use in generated files                        |
 | `version`                        | string   | `undefined`       | Global version to include in all generated files              |
 | `customLLMFiles`                 | array    | `[]`              | Array of custom LLM file configurations                       |
+| `generateMarkdownFiles`          | boolean  | `false`           | Generate individual markdown files and link to them from llms.txt |
+| `rootContent`                    | string   | (see below)       | Custom content to include at the root level of llms.txt       |
+| `fullRootContent`                | string   | (see below)       | Custom content to include at the root level of llms-full.txt  |
+
+### Custom Root Content
+
+The `rootContent` and `fullRootContent` options allow you to customize the introductory content that appears in your generated files, following the llmstxt.org standard which allows "zero or more markdown sections (e.g. paragraphs, lists, etc) of any type except headings" after the title and description.
+
+#### Default Content
+
+If not specified, the plugin uses these defaults:
+- **llms.txt**: "This file contains links to documentation sections following the llmstxt.org standard."
+- **llms-full.txt**: "This file contains all documentation content in a single document following the llmstxt.org standard."
+
+#### Custom Content Examples
+
+**Example 1**: Add project-specific context
+```js
+rootContent: `Welcome to the MyProject documentation.
+
+This documentation covers:
+- Installation and setup
+- API reference
+- Advanced usage guides
+- Troubleshooting
+
+For the latest updates, visit https://myproject.dev/changelog`
+```
+
+**Example 2**: Add technical specifications
+```js
+fullRootContent: `Complete offline documentation bundle for MyProject v2.0.
+
+**Format**: Markdown with code examples
+**Languages**: JavaScript, TypeScript, Python
+**Last Generated**: ${new Date().toISOString()}
+
+> Note: Some features require authentication tokens.
+> See the Authentication section for details.`
+```
+
+**Example 3**: Add navigation hints for AI assistants
+```js
+rootContent: `This documentation is optimized for AI assistants and LLMs.
+
+Quick navigation:
+- For API endpoints, search for "API:"
+- For code examples, search for "Example:"
+- For configuration, search for "Config:"
+
+All code examples are MIT licensed unless otherwise noted.`
+```
+
+#### Custom Root Content for Custom LLM Files
+
+You can also specify root content for each custom LLM file:
+
+```js
+customLLMFiles: [
+  {
+    filename: 'llms-api.txt',
+    includePatterns: ['api/**/*.md'],
+    fullContent: true,
+    title: 'API Documentation',
+    rootContent: `Complete API reference for all REST endpoints.
+    
+Authentication required for all endpoints except /health.
+Base URL: https://api.example.com/v2`
+  }
+]
+```
 
 ### Path Transformation Examples
 
@@ -194,6 +276,46 @@ includeOrder: [
 ```
 Result: Installation and quick-start guides appear first, followed by other getting-started files, then API documentation in a specific order.
 
+### Docusaurus Partials Support
+
+The plugin fully supports [Docusaurus partials](https://docusaurus.io/docs/markdown-features/react#importing-markdown) - reusable MDX content files that can be imported into other documents.
+
+#### How It Works
+
+1. **Partial files** (MDX files starting with underscore, e.g., `_shared-config.mdx`) are automatically excluded from the generated `llms*.txt` files
+2. **Import statements** for partials are resolved and the content is inlined when processing documents
+
+#### Example
+
+Given a partial file `_api-config.mdx`:
+```mdx
+## API Configuration
+
+Set your API endpoint:
+```javascript
+const API_URL = 'https://api.example.com';
+```
+```
+
+And a document that imports it:
+```mdx
+---
+title: Getting Started
+---
+
+# Getting Started Guide
+
+import ApiConfig from './_api-config.mdx';
+
+<ApiConfig />
+
+Now you can make API calls...
+```
+
+The plugin will:
+- Exclude `_api-config.mdx` from `llms.txt`
+- Replace the import and `<ApiConfig />` with the actual content in the processed document
+
 ### Custom LLM Files
 
 In addition to the standard `llms.txt` and `llms-full.txt` files, you can generate custom LLM-friendly files for different sections of your documentation with the `customLLMFiles` option:
@@ -345,9 +467,366 @@ The generated files will include the version information under the description:
 
 Version: 1.0.0
 
-This file contains all documentation content in a single document following the llmtxt.org standard.
+This file contains all documentation content in a single document following the llmstxt.org standard.
+```
+
+## Content Cleaning Options
+
+The plugin provides advanced content cleaning options to optimize your documentation for LLM consumption by removing unnecessary elements that can clutter the output.
+
+### Import Statement Removal (`excludeImports`)
+
+The `excludeImports` option removes JavaScript/TypeScript import statements from your MDX files, which are typically not useful for LLMs and can create noise in the generated documentation.
+
+#### When to Use
+- Your documentation uses MDX files with React components
+- You have many import statements for UI components
+- You want cleaner, more readable output for LLMs
+
+#### Example
+
+**Before** (with `excludeImports: false`):
+```markdown
+import ApiTabs from "@theme/ApiTabs";
+import DiscriminatorTabs from "@theme/DiscriminatorTabs";
+import MethodEndpoint from "@theme/ApiExplorer/MethodEndpoint";
+import SecuritySchemes from "@theme/ApiExplorer/SecuritySchemes";
+import MimeTabs from "@theme/MimeTabs";
+import ParamsItem from "@theme/ParamsItem";
+
+# Create User Account
+
+This endpoint creates a new user account...
+```
+
+**After** (with `excludeImports: true`):
+```markdown
+# Create User Account
+
+This endpoint creates a new user account...
+```
+
+#### Configuration
+```js
+{
+  excludeImports: true, // Remove all import statements
+}
+```
+
+### Duplicate Heading Removal (`removeDuplicateHeadings`)
+
+The `removeDuplicateHeadings` option removes redundant content that simply repeats the heading text immediately after the heading, which is common in auto-generated API documentation.
+
+#### When to Use
+- Your documentation has redundant content that repeats heading text
+- You have auto-generated API docs with minimal content
+- You want to eliminate repetitive patterns for cleaner LLM consumption
+
+#### Example
+
+**Before** (with `removeDuplicateHeadings: false`):
+```markdown
+# Create Deliverable
+
+Create Deliverable
+
+---
+
+# Update User Profile
+
+Update User Profile
+
+---
+```
+
+**After** (with `removeDuplicateHeadings: true`):
+```markdown
+# Create Deliverable
+
+---
+
+# Update User Profile
+
+---
+```
+
+#### Configuration
+```js
+{
+  removeDuplicateHeadings: true, // Remove redundant heading text
+}
+```
+
+### Combined Content Cleaning
+
+For optimal LLM-friendly output, you can combine both options:
+
+```js
+module.exports = {
+  plugins: [
+    [
+      'docusaurus-plugin-llms',
+      {
+        // Enable both content cleaning options for optimal LLM output
+        excludeImports: true,
+        removeDuplicateHeadings: true,
+        
+        // Other configuration options...
+        generateLLMsTxt: true,
+        generateLLMsFullTxt: true,
+        docsDir: 'docs',
+      },
+    ],
+  ],
+};
+```
+
+### Content Cleaning by Use Case
+
+#### Minimal Cleanup (Default Behavior)
+```js
+{
+  excludeImports: false,
+  removeDuplicateHeadings: false
+}
+```
+- Preserves all original content
+- Suitable when you want to keep import statements for reference
+- Good for documentation that doesn't have redundant patterns
+
+#### Import Cleanup Only
+```js
+{
+  excludeImports: true,
+  removeDuplicateHeadings: false
+}
+```
+- Removes import statements but keeps all content
+- Good for MDX-heavy documentation sites
+- Maintains content structure while removing technical imports
+
+#### Full Cleanup (Recommended for LLMs)
+```js
+{
+  excludeImports: true,
+  removeDuplicateHeadings: true
+}
+```
+- Maximum cleanup for LLM consumption
+- Removes both imports and redundant content
+- Recommended for API documentation and auto-generated content
+- Produces the cleanest, most concise output
+
+## Best Practices
+
+### For API Documentation
+If you have auto-generated API documentation (like OpenAPI docs), enable both cleaning options:
+
+```js
+{
+  excludeImports: true,           // Remove React component imports
+  removeDuplicateHeadings: true,  // Remove redundant API endpoint descriptions
+  generateLLMsFullTxt: true,      // Create comprehensive single file
+}
+```
+
+### For Tutorial Content
+For hand-written tutorials and guides, you might want selective cleaning:
+
+```js
+{
+  excludeImports: true,           // Remove any MDX imports
+  removeDuplicateHeadings: false, // Keep all content as written
+  includeOrder: [                // Organize content logically
+    'getting-started/*',
+    'tutorials/*',
+    'advanced/*'
+  ]
+}
+```
+
+### For Multi-Language Documentation
+Create separate clean files for different programming languages:
+
+```js
+{
+  excludeImports: true,
+  removeDuplicateHeadings: true,
+  customLLMFiles: [
+    {
+      filename: 'llms-python.txt',
+      includePatterns: ['**/python/**/*.md'],
+      fullContent: true,
+      title: 'Python Documentation'
+    },
+    {
+      filename: 'llms-javascript.txt', 
+      includePatterns: ['**/javascript/**/*.md'],
+      fullContent: true,
+      title: 'JavaScript Documentation'
+    }
+  ]
+}
+```
+
+### Performance Considerations
+- Content cleaning adds minimal processing overhead
+- Both options work on the content after HTML tag removal
+- No impact on your site's build performance
+- Cleaning happens only during the LLM file generation phase
+
+### Backward Compatibility
+Both options default to `false`, ensuring existing configurations continue to work without changes. Only users who explicitly enable these features will see the cleaned output.
+
+## Markdown File Generation (`generateMarkdownFiles`)
+
+The `generateMarkdownFiles` option enables the plugin to generate individual markdown files for each documentation page, following the [llmstxt.org specification](https://llmstxt.org/) more closely. When enabled, this creates separate `.md` files for LLM consumption instead of linking to your original documentation pages.
+
+### How It Works
+
+**Default Behavior (generateMarkdownFiles: false)**:
+- Generates `llms.txt` with links to your original documentation pages
+- Example: `[Getting Started](https://yoursite.com/docs/getting-started)`
+
+**With generateMarkdownFiles: true**:
+- Generates individual markdown files (e.g., `getting-started.md`, `api-reference.md`)
+- Generates `llms.txt` with links to these generated markdown files  
+- Example: `[Getting Started](https://yoursite.com/getting-started.md)`
+
+### Key Benefits
+
+1. **Standards Compliance**: Follows the llmstxt.org specification by providing individual markdown files rather than linking to HTML pages
+2. **LLM Optimization**: Generated files contain clean, processed markdown optimized for LLM consumption
+3. **Self-Contained**: All necessary content is available in markdown format without requiring HTML parsing
+4. **Flexible Naming**: Automatically generates readable filenames based on document titles
+
+### Configuration Example
+
+```js
+module.exports = {
+  plugins: [
+    [
+      'docusaurus-plugin-llms',
+      {
+        generateMarkdownFiles: true,  // Enable individual markdown file generation
+        generateLLMsTxt: true,        // Generate index file linking to markdown files
+        excludeImports: true,         // Clean up import statements  
+        removeDuplicateHeadings: true, // Remove redundant content
+        
+        // Other options work normally
+        includeOrder: ['getting-started/*', 'guides/*', 'api/*'],
+        pathTransformation: {
+          ignorePaths: ['docs']
+        }
+      }
+    ]
+  ]
+}
+```
+
+### Generated File Structure
+
+With `generateMarkdownFiles: true`, your output directory will contain:
+
+```
+build/
+├── llms.txt              # Index file with links to generated markdown files
+├── llms-full.txt         # Full content file (if enabled)
+├── getting-started.md    # Generated from your getting started docs
+├── api-reference.md      # Generated from your API documentation  
+├── user-guide.md         # Generated from your user guides
+└── ...                   # Other generated markdown files
 ```
 
+### Filename Generation
+
+The plugin generates readable filenames using this priority:
+
+1. **Document Title**: Converted to kebab-case (e.g., "Getting Started" → `getting-started.md`)
+2. **URL Path**: If title is unavailable, uses the document's URL path
+3. **Uniqueness**: Automatically appends numbers for duplicate names (e.g., `getting-started-1.md`)
+
+### Content Processing
+
+Generated markdown files include:
+
+- **Document title** as H1 heading
+- **Document description** as blockquote (following llmstxt.org format)
+- **Processed content** with optional cleaning (import removal, duplicate heading removal)
+- **Proper markdown formatting** optimized for LLM consumption
+
+### Example Generated File
+
+Input documentation about "API Authentication" would generate `api-authentication.md`:
+
+```markdown
+# API Authentication
+
+> Learn how to authenticate with our API using various methods
+
+## Overview
+
+This guide covers all authentication methods supported by our API...
+
+## API Key Authentication
+
+Use your API key to authenticate requests:
+
+```javascript
+const client = new Client({ apiKey: 'your-key' });
+```
+```
+
+### Use Cases
+
+#### Standards-Compliant Documentation
+Perfect for projects that want to follow the llmstxt.org specification exactly:
+
+```js
+{
+  generateMarkdownFiles: true,
+  generateLLMsTxt: true,
+  generateLLMsFullTxt: false  // Optional: disable if only individual files are needed
+}
+```
+
+#### LLM Training Data
+Generate clean markdown files for LLM training or fine-tuning:
+
+```js
+{
+  generateMarkdownFiles: true,
+  excludeImports: true,
+  removeDuplicateHeadings: true,
+  customLLMFiles: [
+    {
+      filename: 'training-data.txt',
+      includePatterns: ['**/*.md'],
+      fullContent: true
+    }
+  ]
+}
+```
+
+#### Multi-Format Output
+Generate both original links and markdown files for different use cases:
+
+```js
+{
+  generateLLMsTxt: true,      // Links to original pages  
+  generateMarkdownFiles: true, // Also generate individual markdown files
+  llmsTxtFilename: 'llms-original.txt',  // Original links file
+  // The markdown-linked version will be in llms.txt
+}
+```
+
+### Compatibility
+
+- **Fully backward compatible**: Defaults to `false`, existing configurations unchanged
+- **Works with all existing options**: Path transformations, custom LLM files, content cleaning
+- **Respects ordering**: Generated files maintain the same order as configured with `includeOrder`
+- **Custom LLM files**: Also support markdown file generation when the global option is enabled
+
 ## How It Works
 
 This plugin automatically generates the following files during the build process:

package/lib/generator.d.ts

@@ -10,8 +10,17 @@ import { DocInfo, PluginContext } from './types';
  * @param fileDescription - Description for the file
  * @param includeFullContent - Whether to include full content or just links
  * @param version - Version of the file
+ * @param customRootContent - Optional custom content to include at the root level
  */
-export declare function generateLLMFile(docs: DocInfo[], outputPath: string, fileTitle: string, fileDescription: string, includeFullContent: boolean, version?: string): Promise<void>;
+export declare function generateLLMFile(docs: DocInfo[], outputPath: string, fileTitle: string, fileDescription: string, includeFullContent: boolean, version?: string, customRootContent?: string): Promise<void>;
+/**
+ * Generate individual markdown files for each document
+ * @param docs - Processed document information
+ * @param outputDir - Directory to write the markdown files
+ * @param siteUrl - Base site URL
+ * @returns Updated docs with new URLs pointing to generated markdown files
+ */
+export declare function generateIndividualMarkdownFiles(docs: DocInfo[], outputDir: string, siteUrl: string): Promise<DocInfo[]>;
 /**
  * Generate standard LLM files (llms.txt and llms-full.txt)
  * @param context - Plugin context