docusaurus-plugin-llms 0.2.2 → 0.3.0

package/README.md

@@ -137,9 +137,12 @@ module.exports = {
 | `version`                        | string   | `undefined`       | Global version to include in all generated files              |
 | `customLLMFiles`                 | array    | `[]`              | Array of custom LLM file configurations                       |
 | `generateMarkdownFiles`          | boolean  | `false`           | Generate individual markdown files and link to them from llms.txt |
-| `keepFrontMatter`                | string[] | []                | Preserve selected front matter items when generating individual markdown files
+| `keepFrontMatter`                | string[] | []                | Preserve selected front matter items when generating individual markdown files |
+| `preserveDirectoryStructure`     | boolean  | `true`            | Preserve full directory structure in generated markdown files (e.g., `docs/server/config.md` instead of `server/config.md`) |
+| `processingBatchSize`            | number   | `100`             | Batch size for processing documents to prevent out-of-memory errors on large sites |
 | `rootContent`                    | string   | (see below)       | Custom content to include at the root level of llms.txt       |
 | `fullRootContent`                | string   | (see below)       | Custom content to include at the root level of llms-full.txt  |
+| `logLevel`                       | string   | `'normal'`        | Logging level for plugin output: `'quiet'`, `'normal'`, or `'verbose'` |
 
 ### Custom Root Content
 
@@ -209,6 +212,37 @@ Base URL: https://api.example.com/v2`
 ]
 ```
 
+### Batch Processing for Large Sites
+
+The plugin includes batch processing to prevent out-of-memory errors when processing very large documentation sites. By default, documents are processed in batches of 100, but you can configure this using the `processingBatchSize` option.
+
+**When to adjust batch size:**
+- **Large sites (1000+ documents)**: Reduce batch size (e.g., `50`) to lower memory usage
+- **Small sites (< 100 documents)**: Default value is fine
+- **Memory-constrained environments**: Reduce batch size to prevent OOM errors
+- **High-memory systems**: Increase batch size (e.g., `200`) for faster processing
+
+**Example configuration:**
+```js
+module.exports = {
+  plugins: [
+    [
+      'docusaurus-plugin-llms',
+      {
+        processingBatchSize: 50, // Process 50 documents at a time
+        // ... other options
+      },
+    ],
+  ],
+};
+```
+
+**How it works:**
+- Documents are processed in chunks of the specified batch size
+- Each batch is processed sequentially to control memory usage
+- Document order is preserved across batches
+- Progress is logged when processing multiple batches (in verbose mode)
+
 ### Path Transformation Examples
 
 The path transformation feature allows you to manipulate how URLs are constructed from file paths:
@@ -242,15 +276,32 @@ The configuration supports multiple path segments in both arrays.
 
 ### Document Ordering Examples
 
-The document ordering feature allows you to control the sequence in which files appear in the generated output:
+The document ordering feature allows you to control the sequence in which files appear in the generated output.
+
+#### Pattern Matching Behavior
+
+Patterns in `includeOrder`, `ignoreFiles`, and `customLLMFiles.includePatterns` are matched against **both** site-relative and docs-relative paths for maximum flexibility:
+
+- **Site-relative path**: The path relative to your site root (e.g., `docs/quickstart/file.md`)
+- **Docs-relative path**: The path relative to your docs directory (e.g., `quickstart/file.md`)
+
+This means both of these patterns will match the same file:
+```js
+includeOrder: [
+  'docs/quickstart/*',      // Matches site-relative path
+  'quickstart/*'            // Matches docs-relative path (more intuitive!)
+]
+```
+
+**Recommended approach**: Use docs-relative paths (without the `docs/` prefix) as they are more intuitive and portable across different configurations.
 
 **Example 1**: Basic Section Ordering
 ```js
 includeOrder: [
-  'getting-started/*',
-  'guides/*',
-  'api/*',
-  'advanced/*'
+  'getting-started/*',  // Matches docs/getting-started/*.md
+  'guides/*',           // Matches docs/guides/*.md
+  'api/*',              // Matches docs/api/*.md
+  'advanced/*'          // Matches docs/advanced/*.md
 ]
 ```
 Result: Files will appear in the generated output following this section order.
@@ -258,7 +309,7 @@ Result: Files will appear in the generated output following this section order.
 **Example 2**: Strict Inclusion List
 ```js
 includeOrder: [
-  'public-docs/**/*.md'
+  'public-docs/**/*.md'  // Matches docs/public-docs/**/*.md
 ],
 includeUnmatchedLast: false
 ```
@@ -267,16 +318,26 @@ Result: Only files matching 'public-docs/**/*.md' are included, all others are e
 **Example 3**: Detailed Ordering with Specific Files First
 ```js
 includeOrder: [
-  'getting-started/installation.md',
-  'getting-started/quick-start.md',
-  'getting-started/*.md',
-  'api/core/*.md',
-  'api/plugins/*.md',
-  'api/**/*.md'
+  'getting-started/installation.md',    // Specific file first
+  'getting-started/quick-start.md',     // Another specific file
+  'getting-started/*.md',               // Rest of getting-started
+  'api/core/*.md',                      // Core API docs
+  'api/plugins/*.md',                   // Plugin API docs
+  'api/**/*.md'                         // All other API docs
 ]
 ```
 Result: Installation and quick-start guides appear first, followed by other getting-started files, then API documentation in a specific order.
 
+**Example 4**: Nested Directory Patterns
+```js
+includeOrder: [
+  'tutorials/beginner/**/*',    // All beginner tutorials (deeply nested)
+  'tutorials/intermediate/*',   // Intermediate tutorials (one level)
+  'tutorials/**/*'              // All other tutorials
+]
+```
+Result: Beginner tutorials appear first (regardless of nesting depth), then intermediate, then everything else.
+
 ### Docusaurus Partials Support
 
 The plugin fully supports [Docusaurus partials](https://docusaurus.io/docs/markdown-features/react#importing-markdown) - reusable MDX content files that can be imported into other documents.
@@ -471,6 +532,130 @@ Version: 1.0.0
 This file contains all documentation content in a single document following the llmstxt.org standard.
 ```
 
+## Logging Configuration
+
+The plugin includes a configurable logging system that allows you to control the amount of output during the build process.
+
+### Log Levels
+
+The plugin supports three logging levels:
+
+- **quiet**: Suppresses all output except errors
+- **normal** (default): Shows standard informational messages and warnings
+- **verbose**: Shows detailed progress information including file-by-file processing
+
+### Configuration
+
+```js
+module.exports = {
+  plugins: [
+    [
+      'docusaurus-plugin-llms',
+      {
+        logLevel: 'verbose', // Options: 'quiet', 'normal', 'verbose'
+        // Other configuration options...
+      },
+    ],
+  ],
+};
+```
+
+### Log Level Details
+
+#### Quiet Mode (`logLevel: 'quiet'`)
+
+Only errors are displayed. Use this for clean builds in CI/CD environments or when you don't need build feedback.
+
+```js
+{
+  logLevel: 'quiet'
+}
+```
+
+Output:
+```
+[docusaurus-plugin-llms] ERROR: Error generating LLM documentation: ...
+```
+
+#### Normal Mode (`logLevel: 'normal'`) - Default
+
+Shows standard progress messages, warnings, and errors. This is the recommended setting for most users.
+
+```js
+{
+  logLevel: 'normal' // or omit - this is the default
+}
+```
+
+Output:
+```
+[docusaurus-plugin-llms] Generating LLM-friendly documentation...
+[docusaurus-plugin-llms] Generating individual markdown files...
+[docusaurus-plugin-llms] Generated: /path/to/llms.txt
+[docusaurus-plugin-llms] Generated: /path/to/llms-full.txt
+[docusaurus-plugin-llms] Stats: 42 total available documents processed
+```
+
+#### Verbose Mode (`logLevel: 'verbose'`)
+
+Shows detailed information about every file being processed. Use this for debugging or when you need detailed feedback.
+
+```js
+{
+  logLevel: 'verbose'
+}
+```
+
+Output:
+```
+[docusaurus-plugin-llms] Generating LLM-friendly documentation...
+[docusaurus-plugin-llms] Generating file: /path/to/llms.txt, version: undefined
+[docusaurus-plugin-llms] Processed 42 documentation files for standard LLM files
+[docusaurus-plugin-llms] Generating individual markdown files...
+[docusaurus-plugin-llms] Generated markdown file: getting-started.md
+[docusaurus-plugin-llms] Generated markdown file: api/reference.md
+[docusaurus-plugin-llms] Generated: /path/to/llms.txt
+[docusaurus-plugin-llms] Generated: /path/to/llms-full.txt
+[docusaurus-plugin-llms] Stats: 42 total available documents processed
+```
+
+### Use Cases
+
+#### Development
+
+Use normal or verbose mode during development to see what's being generated:
+
+```js
+{
+  logLevel: 'verbose',
+  generateMarkdownFiles: true
+}
+```
+
+#### Production/CI
+
+Use quiet mode in production builds or CI/CD to reduce log noise:
+
+```js
+{
+  logLevel: 'quiet',
+  generateLLMsTxt: true,
+  generateLLMsFullTxt: true
+}
+```
+
+#### Debugging
+
+Use verbose mode when troubleshooting issues:
+
+```js
+{
+  logLevel: 'verbose',
+  excludeImports: true,
+  removeDuplicateHeadings: true
+}
+```
+
 ## Content Cleaning Options
 
 The plugin provides advanced content cleaning options to optimize your documentation for LLM consumption by removing unnecessary elements that can clutter the output.
@@ -725,6 +910,35 @@ module.exports = {
 }
 ```
 
+### Directory Structure Options
+
+#### Preserving Directory Structure (`preserveDirectoryStructure`)
+
+By default (`preserveDirectoryStructure: true`), generated markdown files maintain the same directory structure as your HTML output, making them accessible at matching URL paths:
+
+**With `preserveDirectoryStructure: true` (default)**:
+```
+docs/server/config.md → build/docs/server/config.md
+```
+
+**With `preserveDirectoryStructure: false`**:
+```
+docs/server/config.md → build/server/config.md
+```
+
+This is particularly useful when you want markdown files to sit alongside HTML files in the build output, allowing them to be served from the same URL path with a `.md` extension.
+
+**Example configuration**:
+```js
+{
+  generateMarkdownFiles: true,
+  preserveDirectoryStructure: true,  // Default: matches HTML output structure
+  docsDir: 'docs'
+}
+```
+
+With this configuration, if your HTML is at `https://yoursite.com/docs/server/config.html`, the markdown will be at `https://yoursite.com/docs/server/config.md`.
+
 ### Generated File Structure
 
 With `generateMarkdownFiles: true`, your output directory will contain:
@@ -733,12 +947,28 @@ With `generateMarkdownFiles: true`, your output directory will contain:
 build/
 ├── llms.txt              # Index file with links to generated markdown files
 ├── llms-full.txt         # Full content file (if enabled)
-├── getting-started.md    # Generated from your getting started docs
-├── api-reference.md      # Generated from your API documentation  
-├── user-guide.md         # Generated from your user guides
+├── docs/                 # Preserves directory structure (default)
+│   ├── getting-started.md
+│   ├── api/
+│   │   └── reference.md
+│   └── server/
+│       └── config.md
 └── ...                   # Other generated markdown files
 ```
 
+Or with `preserveDirectoryStructure: false`:
+
+```
+build/
+├── llms.txt              # Index file with links to generated markdown files
+├── llms-full.txt         # Full content file (if enabled)
+├── getting-started.md    # Flat structure (old behavior)
+├── api/
+│   └── reference.md
+└── server/
+    └── config.md
+```
+
 ### Filename Generation
 
 The plugin generates readable filenames using this priority:

package/lib/generator-current.d.ts

@@ -0,0 +1,44 @@
+/**
+ * LLM file generation functions for the docusaurus-plugin-llms plugin
+ */
+import { DocInfo, PluginContext } from './types';
+/**
+ * Generate an LLM-friendly file
+ * @param docs - Processed document information
+ * @param outputPath - Path to write the output file
+ * @param fileTitle - Title for the file
+ * @param fileDescription - Description for the file
+ * @param includeFullContent - Whether to include full content or just links
+ * @param version - Version of the file
+ * @param customRootContent - Optional custom content to include at the root level
+ */
+export declare function generateLLMFile(docs: DocInfo[], outputPath: string, fileTitle: string, fileDescription: string, includeFullContent: boolean, version?: string, customRootContent?: string): Promise<void>;
+/**
+ * Generate individual markdown files for each document
+ * @param docs - Processed document information
+ * @param outputDir - Directory to write the markdown files
+ * @param siteUrl - Base site URL
+ * @param docsDir - The configured docs directory name (e.g., 'docs', 'documentation', etc.)
+ * @param keepFrontMatter - Array of frontmatter keys to preserve in generated files
+ * @param preserveDirectoryStructure - Whether to preserve the full directory structure (default: true)
+ * @returns Updated docs with new URLs pointing to generated markdown files
+ */
+export declare function generateIndividualMarkdownFiles(docs: DocInfo[], outputDir: string, siteUrl: string, docsDir?: string, keepFrontMatter?: string[], preserveDirectoryStructure?: boolean): Promise<DocInfo[]>;
+/**
+ * Generate standard LLM files (llms.txt and llms-full.txt)
+ * @param context - Plugin context
+ * @param allDocFiles - Array of all document files
+ */
+export declare function generateStandardLLMFiles(context: PluginContext, allDocFiles: string[]): Promise<void>;
+/**
+ * Generate custom LLM files based on configuration
+ * @param context - Plugin context
+ * @param allDocFiles - Array of all document files
+ */
+export declare function generateCustomLLMFiles(context: PluginContext, allDocFiles: string[]): Promise<void>;
+/**
+ * Collect all document files from docs directory and optionally blog
+ * @param context - Plugin context
+ * @returns Array of file paths
+ */
+export declare function collectDocFiles(context: PluginContext): Promise<string[]>;