docusaurus-plugin-llms 0.1.2 → 0.1.3

package/README.md

@@ -39,6 +39,13 @@ module.exports = {
         title: 'My Project Documentation',
         description: 'Complete reference documentation for My Project',
         includeBlog: true,
+        // Control documentation order
+        includeOrder: [
+          'getting-started/*',
+          'guides/*',
+          'api/*',
+        ],
+        includeUnmatchedLast: true,
         // Path transformation options
         pathTransformation: {
           // Paths to ignore when constructing URLs (will be removed if found)
@@ -46,6 +53,23 @@ module.exports = {
           // Paths to add when constructing URLs (will be prepended if not already present)
           addPaths: ['api'],
         },
+        // Custom LLM files for specific documentation sections
+        customLLMFiles: [
+          {
+            filename: 'llms-python.txt',
+            includePatterns: ['api/python/**/*.md', 'guides/python/*.md'],
+            fullContent: true,
+            title: 'Python API Documentation',
+            description: 'Complete reference for Python API'
+          },
+          {
+            filename: 'llms-tutorials.txt',
+            includePatterns: ['tutorials/**/*.md'],
+            fullContent: false,
+            title: 'Tutorial Documentation',
+            description: 'All tutorials in a single file'
+          }
+        ],
       },
     ],
     // ... your other plugins
@@ -55,20 +79,24 @@ module.exports = {
 
 ### Available Options
 
-| Option                | Type     | Default           | Description                                  |
-|-----------------------|----------|-------------------|----------------------------------------------|
-| `generateLLMsTxt`     | boolean  | `true`            | Whether to generate the links file           |
-| `generateLLMsFullTxt` | boolean  | `true`            | Whether to generate the full content file    |
-| `docsDir`             | string   | `'docs'`          | Base directory for documentation files       |
-| `ignoreFiles`         | string[] | `[]`              | Array of glob patterns for files to ignore   |
-| `title`               | string   | Site title        | Custom title to use in generated files       |
-| `description`         | string   | Site tagline      | Custom description to use in generated files |
-| `llmsTxtFilename`     | string   | `'llms.txt'`      | Custom filename for the links file           |
-| `llmsFullTxtFilename` | string   | `'llms-full.txt'` | Custom filename for the full content file    |
-| `includeBlog`         | boolean  | `false`           | Whether to include blog content              |
-| `pathTransformation`  | object   | `undefined`       | Path transformation options for URL construction |
-| `pathTransformation.ignorePaths` | string[] | `[]`    | Path segments to ignore when constructing URLs |
-| `pathTransformation.addPaths`   | string[] | `[]`    | Path segments to add when constructing URLs |
+| Option                           | Type     | Default           | Description                                                   |
+|----------------------------------|----------|-------------------|---------------------------------------------------------------|
+| `description`                    | string   | Site tagline      | Custom description to use in generated files                  |
+| `docsDir`                        | string   | `'docs'`          | Base directory for documentation files                        |
+| `generateLLMsFullTxt`            | boolean  | `true`            | Whether to generate the full content file                     |
+| `generateLLMsTxt`                | boolean  | `true`            | Whether to generate the links file                            |
+| `ignoreFiles`                    | string[] | `[]`              | Array of glob patterns for files to ignore                    |
+| `includeBlog`                    | boolean  | `false`           | Whether to include blog content                               |
+| `includeOrder`                   | string[] | `[]`              | Array of glob patterns for files to process in specific order |
+| `includeUnmatchedLast`           | boolean  | `true`            | Whether to include unmatched files at the end                 |
+| `llmsFullTxtFilename`            | string   | `'llms-full.txt'` | Custom filename for the full content file                     |
+| `llmsTxtFilename`                | string   | `'llms.txt'`      | Custom filename for the links file                            |
+| `pathTransformation.addPaths`    | string[] | `[]`              | Path segments to add when constructing URLs                   |
+| `pathTransformation.ignorePaths` | string[] | `[]`              | Path segments to ignore when constructing URLs                |
+| `pathTransformation`             | object   | `undefined`       | Path transformation options for URL construction              |
+| `title`                          | string   | Site title        | Custom title to use in generated files                        |
+| `version`                        | string   | `undefined`       | Global version to include in all generated files              |
+| `customLLMFiles`                 | array    | `[]`              | Array of custom LLM file configurations                       |
 
 ### Path Transformation Examples
 
@@ -101,12 +129,204 @@ File path: `/content/docs/manual/decorators.md` → URL: `https://example.com/ap
 
 The configuration supports multiple path segments in both arrays.
 
+### Document Ordering Examples
+
+The document ordering feature allows you to control the sequence in which files appear in the generated output:
+
+**Example 1**: Basic Section Ordering
+```js
+includeOrder: [
+  'getting-started/*',
+  'guides/*',
+  'api/*',
+  'advanced/*'
+]
+```
+Result: Files will appear in the generated output following this section order.
+
+**Example 2**: Strict Inclusion List
+```js
+includeOrder: [
+  'public-docs/**/*.md'
+],
+includeUnmatchedLast: false
+```
+Result: Only files matching 'public-docs/**/*.md' are included, all others are excluded.
+
+**Example 3**: Detailed Ordering with Specific Files First
+```js
+includeOrder: [
+  'getting-started/installation.md',
+  'getting-started/quick-start.md',
+  'getting-started/*.md',
+  'api/core/*.md',
+  'api/plugins/*.md',
+  'api/**/*.md'
+]
+```
+Result: Installation and quick-start guides appear first, followed by other getting-started files, then API documentation in a specific order.
+
+### Custom LLM Files
+
+In addition to the standard `llms.txt` and `llms-full.txt` files, you can generate custom LLM-friendly files for different sections of your documentation with the `customLLMFiles` option:
+
+```js
+customLLMFiles: [
+  {
+    filename: 'llms-python.txt',
+    includePatterns: ['api/python/**/*.md', 'guides/python/*.md'],
+    fullContent: true,
+    title: 'Python API Documentation',
+    description: 'Complete reference for Python API'
+  },
+  {
+    filename: 'llms-tutorials.txt',
+    includePatterns: ['tutorials/**/*.md'],
+    fullContent: false,
+    title: 'Tutorial Documentation',
+    description: 'All tutorials in a single file'
+  }
+]
+```
+
+#### Custom LLM File Configuration
+
+Each custom LLM file is defined by an object with the following properties:
+
+| Option                | Type     | Required | Description                                  |
+|-----------------------|----------|----------|----------------------------------------------|
+| `filename`            | string   | Yes      | Name of the output file (e.g., 'llms-python.txt') |
+| `includePatterns`     | string[] | Yes      | Glob patterns for files to include |
+| `fullContent`         | boolean  | Yes      | `true` for full content like llms-full.txt, `false` for links only like llms.txt |
+| `title`               | string   | No       | Custom title for this file (defaults to site title) |
+| `description`         | string   | No       | Custom description for this file (defaults to site description) |
+| `ignorePatterns`      | string[] | No       | Additional patterns to exclude (combined with global ignoreFiles) |
+| `orderPatterns`       | string[] | No       | Order patterns for controlling file ordering (similar to includeOrder) |
+| `includeUnmatchedLast`| boolean  | No       | Whether to include unmatched files last (default: false) |
+| `version`             | string   | No       | Version information for this LLM file (overrides global version) |
+
+#### Use Cases
+
+##### Language-Specific Documentation
+
+Create separate files for different programming languages:
+
+```js
+customLLMFiles: [
+  {
+    filename: 'llms-python.txt',
+    includePatterns: ['api/python/**/*.md', 'guides/python/*.md'],
+    fullContent: true,
+    title: 'Python API Documentation'
+  },
+  {
+    filename: 'llms-javascript.txt',
+    includePatterns: ['api/javascript/**/*.md', 'guides/javascript/*.md'],
+    fullContent: true,
+    title: 'JavaScript API Documentation'
+  }
+]
+```
+
+##### Content Type Separation
+
+Separate tutorials from API reference:
+
+```js
+customLLMFiles: [
+  {
+    filename: 'llms-tutorials.txt',
+    includePatterns: ['tutorials/**/*.md', 'guides/**/*.md'],
+    fullContent: true,
+    title: 'Tutorials and Guides'
+  },
+  {
+    filename: 'llms-api.txt',
+    includePatterns: ['api/**/*.md', 'reference/**/*.md'],
+    fullContent: true,
+    title: 'API Reference'
+  }
+]
+```
+
+##### Beginner-Friendly Documentation
+
+Create a beginner-focused file with carefully ordered content:
+
+```js
+customLLMFiles: [
+  {
+    filename: 'llms-getting-started.txt',
+    includePatterns: ['**/*.md'],
+    ignorePatterns: ['advanced/**/*.md', 'internal/**/*.md'],
+    orderPatterns: [
+      'introduction.md',
+      'getting-started/*.md',
+      'tutorials/basic/*.md',
+      'examples/simple/*.md'
+    ],
+    fullContent: true,
+    title: 'Getting Started Guide',
+    description: 'Beginner-friendly documentation with essential concepts'
+  }
+]
+```
+
+##### Versioned Documentation
+
+Include version information in your documentation files:
+
+```js
+plugins: [
+  [
+    'docusaurus-plugin-llms',
+    {
+      // Global version applies to all files
+      version: '2.0.0',
+      
+      // Custom LLM files with specific versions
+      customLLMFiles: [
+        {
+          filename: 'api-reference.txt',
+          title: 'API Reference Documentation',
+          description: 'Complete API reference for developers',
+          includePatterns: ['**/api/**/*.md', '**/reference/**/*.md'],
+          fullContent: true,
+          version: '1.0.0'  // Overrides global version
+        },
+        {
+          filename: 'tutorials.txt',
+          title: 'Tutorials and Guides',
+          description: 'Step-by-step tutorials and guides',
+          includePatterns: ['**/tutorials/**/*.md', '**/guides/**/*.md'],
+          fullContent: true,
+          version: '0.9.5-beta'  // Overrides global version
+        }
+      ]
+    }
+  ],
+]
+```
+
+The generated files will include the version information under the description:
+
+```
+# API Reference Documentation
+
+> Complete API reference for developers
+
+Version: 1.0.0
+
+This file contains all documentation content in a single document following the llmtxt.org standard.
+```
+
 ## How It Works
 
 This plugin automatically generates the following files during the build process:
 
 - **llms.txt**: Contains links to all sections of your documentation
 - **llms-full.txt**: Contains all documentation content in a single file
+- **Custom LLM files**: Additional files based on your custom configurations
 
 These files follow the [llmstxt standard](https://llmstxt.org/), making your documentation optimized for use with Large Language Models (LLMs).
 
@@ -117,10 +337,12 @@ These files follow the [llmstxt standard](https://llmstxt.org/), making your doc
 - ⚙️ Highly customizable with multiple options
 - 📝 Creates `llms.txt` with section links
 - 📖 Produces `llms-full.txt` with all content in one file
+- 📋 Document ordering control for custom sequence
+- 🔄 Path transformation to customize URL construction
+- 📚 Option to include blog posts
+- 🧩 Custom LLM files for specific documentation sections
 - 🧹 Cleans HTML and normalizes content for optimal LLM consumption
 - 📊 Provides statistics about generated documentation
-- 📚 Option to include blog posts
-- 🔄 Path transformation to customize URL construction
 
 ## Implementation Details
 
@@ -128,12 +350,14 @@ The plugin:
 
 1. Scans your `docs` directory recursively for all Markdown files
 2. Optionally includes blog content
-3. Extracts metadata, titles, and content from each file
-4. Creates proper URL links to each document section
-5. Applies path transformations according to configuration (removing or adding path segments)
-6. Generates a table of contents in `llms.txt`
-7. Combines all documentation content in `llms-full.txt`
-8. Provides statistics about the generated documentation
+3. Orders documents according to specified glob patterns (if provided)
+4. Extracts metadata, titles, and content from each file
+5. Creates proper URL links to each document section
+6. Applies path transformations according to configuration (removing or adding path segments)
+7. Generates a table of contents in `llms.txt`
+8. Combines all documentation content in `llms-full.txt`
+9. Creates custom LLM files based on specified configurations
+10. Provides statistics about the generated documentation
 
 ## Testing
 

package/lib/generator.d.ts

@@ -0,0 +1,32 @@
+/**
+ * LLM file generation functions for the docusaurus-plugin-llms plugin
+ */
+import { DocInfo, PluginContext } from './types';
+/**
+ * Generate an LLM-friendly file
+ * @param docs - Processed document information
+ * @param outputPath - Path to write the output file
+ * @param fileTitle - Title for the file
+ * @param fileDescription - Description for the file
+ * @param includeFullContent - Whether to include full content or just links
+ * @param version - Version of the file
+ */
+export declare function generateLLMFile(docs: DocInfo[], outputPath: string, fileTitle: string, fileDescription: string, includeFullContent: boolean, version?: string): Promise<void>;
+/**
+ * Generate standard LLM files (llms.txt and llms-full.txt)
+ * @param context - Plugin context
+ * @param allDocFiles - Array of all document files
+ */
+export declare function generateStandardLLMFiles(context: PluginContext, allDocFiles: string[]): Promise<void>;
+/**
+ * Generate custom LLM files based on configuration
+ * @param context - Plugin context
+ * @param allDocFiles - Array of all document files
+ */
+export declare function generateCustomLLMFiles(context: PluginContext, allDocFiles: string[]): Promise<void>;
+/**
+ * Collect all document files from docs directory and optionally blog
+ * @param context - Plugin context
+ * @returns Array of file paths
+ */
+export declare function collectDocFiles(context: PluginContext): Promise<string[]>;

package/lib/generator.js

@@ -0,0 +1,212 @@
+"use strict";
+/**
+ * LLM file generation functions for the docusaurus-plugin-llms plugin
+ */
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.generateLLMFile = generateLLMFile;
+exports.generateStandardLLMFiles = generateStandardLLMFiles;
+exports.generateCustomLLMFiles = generateCustomLLMFiles;
+exports.collectDocFiles = collectDocFiles;
+const path = __importStar(require("path"));
+const fs = __importStar(require("fs/promises"));
+const utils_1 = require("./utils");
+const processor_1 = require("./processor");
+/**
+ * Clean a description for use in a TOC item
+ * @param description - The original description
+ * @returns Cleaned description suitable for TOC
+ */
+function cleanDescriptionForToc(description) {
+    if (!description)
+        return '';
+    // Get just the first line for TOC display
+    const firstLine = description.split('\n')[0];
+    // Remove heading markers only at the beginning of the line
+    // Be careful to only remove actual heading markers (# followed by space at beginning)
+    // and not hashtag symbols that are part of the content (inline hashtags)
+    const cleaned = firstLine.replace(/^(#+)\s+/g, '');
+    // Truncate if too long (150 characters max with ellipsis)
+    return cleaned.length > 150 ? cleaned.substring(0, 147) + '...' : cleaned;
+}
+/**
+ * Generate an LLM-friendly file
+ * @param docs - Processed document information
+ * @param outputPath - Path to write the output file
+ * @param fileTitle - Title for the file
+ * @param fileDescription - Description for the file
+ * @param includeFullContent - Whether to include full content or just links
+ * @param version - Version of the file
+ */
+async function generateLLMFile(docs, outputPath, fileTitle, fileDescription, includeFullContent, version) {
+    console.log(`Generating file: ${outputPath}, version: ${version || 'undefined'}`);
+    const versionInfo = version ? `\n\nVersion: ${version}` : '';
+    if (includeFullContent) {
+        // Generate full content file
+        const fullContentSections = docs.map(doc => {
+            return `## ${doc.title}
+
+${doc.content}`;
+        });
+        const llmFileContent = `# ${fileTitle}
+
+> ${fileDescription}${versionInfo}
+
+This file contains all documentation content in a single document following the llmtxt.org standard.
+
+${fullContentSections.join('\n\n---\n\n')}
+`;
+        await (0, utils_1.writeFile)(outputPath, llmFileContent);
+    }
+    else {
+        // Generate links-only file
+        const tocItems = docs.map(doc => {
+            // Clean and format the description for TOC
+            const cleanedDescription = cleanDescriptionForToc(doc.description);
+            return `- [${doc.title}](${doc.url})${cleanedDescription ? `: ${cleanedDescription}` : ''}`;
+        });
+        const llmFileContent = `# ${fileTitle}
+
+> ${fileDescription}${versionInfo}
+
+This file contains links to documentation sections following the llmtxt.org standard.
+
+## Table of Contents
+
+${tocItems.join('\n')}
+`;
+        await (0, utils_1.writeFile)(outputPath, llmFileContent);
+    }
+    console.log(`Generated: ${outputPath}`);
+}
+/**
+ * Generate standard LLM files (llms.txt and llms-full.txt)
+ * @param context - Plugin context
+ * @param allDocFiles - Array of all document files
+ */
+async function generateStandardLLMFiles(context, allDocFiles) {
+    const { outDir, docTitle, docDescription, options } = context;
+    const { generateLLMsTxt, generateLLMsFullTxt, llmsTxtFilename = 'llms.txt', llmsFullTxtFilename = 'llms-full.txt', includeOrder = [], includeUnmatchedLast = true, version } = options;
+    if (!generateLLMsTxt && !generateLLMsFullTxt) {
+        return;
+    }
+    // Process files for the standard outputs
+    const processedDocs = await (0, processor_1.processFilesWithPatterns)(context, allDocFiles, [], // No specific include patterns - include all
+    [], // No additional ignore patterns beyond global ignoreFiles
+    includeOrder, includeUnmatchedLast);
+    console.log(`Processed ${processedDocs.length} documentation files for standard LLM files`);
+    // Generate llms.txt
+    if (generateLLMsTxt) {
+        const llmsTxtPath = path.join(outDir, llmsTxtFilename);
+        await generateLLMFile(processedDocs, llmsTxtPath, docTitle, docDescription, false, // links only
+        version);
+    }
+    // Generate llms-full.txt
+    if (generateLLMsFullTxt) {
+        const llmsFullTxtPath = path.join(outDir, llmsFullTxtFilename);
+        await generateLLMFile(processedDocs, llmsFullTxtPath, docTitle, docDescription, true, // full content
+        version);
+    }
+}
+/**
+ * Generate custom LLM files based on configuration
+ * @param context - Plugin context
+ * @param allDocFiles - Array of all document files
+ */
+async function generateCustomLLMFiles(context, allDocFiles) {
+    const { outDir, docTitle, docDescription, options } = context;
+    const { customLLMFiles = [], ignoreFiles = [] } = options;
+    if (customLLMFiles.length === 0) {
+        return;
+    }
+    console.log(`Generating ${customLLMFiles.length} custom LLM files...`);
+    for (const customFile of customLLMFiles) {
+        console.log(`Processing custom file: ${customFile.filename}, version: ${customFile.version || 'undefined'}`);
+        // Combine global ignores with custom ignores
+        const combinedIgnores = [...ignoreFiles];
+        if (customFile.ignorePatterns) {
+            combinedIgnores.push(...customFile.ignorePatterns);
+        }
+        // Process files according to the custom configuration
+        const customDocs = await (0, processor_1.processFilesWithPatterns)(context, allDocFiles, customFile.includePatterns, combinedIgnores, customFile.orderPatterns || [], customFile.includeUnmatchedLast ?? false);
+        if (customDocs.length > 0) {
+            // Use custom title/description or fall back to defaults
+            const customTitle = customFile.title || docTitle;
+            const customDescription = customFile.description || docDescription;
+            // Generate the custom LLM file
+            const customFilePath = path.join(outDir, customFile.filename);
+            await generateLLMFile(customDocs, customFilePath, customTitle, customDescription, customFile.fullContent, customFile.version);
+            console.log(`Generated custom LLM file: ${customFile.filename} with ${customDocs.length} documents`);
+        }
+        else {
+            console.warn(`No matching documents found for custom LLM file: ${customFile.filename}`);
+        }
+    }
+}
+/**
+ * Collect all document files from docs directory and optionally blog
+ * @param context - Plugin context
+ * @returns Array of file paths
+ */
+async function collectDocFiles(context) {
+    const { siteDir, docsDir, options } = context;
+    const { ignoreFiles = [], includeBlog = false } = options;
+    const allDocFiles = [];
+    // Process docs directory
+    const fullDocsDir = path.join(siteDir, docsDir);
+    try {
+        await fs.access(fullDocsDir);
+        // Collect all markdown files from docs directory
+        const docFiles = await (0, utils_1.readMarkdownFiles)(fullDocsDir, siteDir, ignoreFiles);
+        allDocFiles.push(...docFiles);
+    }
+    catch (err) {
+        console.warn(`Docs directory not found: ${fullDocsDir}`);
+    }
+    // Process blog if enabled
+    if (includeBlog) {
+        const blogDir = path.join(siteDir, 'blog');
+        try {
+            await fs.access(blogDir);
+            // Collect all markdown files from blog directory
+            const blogFiles = await (0, utils_1.readMarkdownFiles)(blogDir, siteDir, ignoreFiles);
+            allDocFiles.push(...blogFiles);
+        }
+        catch (err) {
+            console.warn(`Blog directory not found: ${blogDir}`);
+        }
+    }
+    return allDocFiles;
+}

package/lib/index.d.ts

@@ -8,36 +8,7 @@
  * The plugin runs during the Docusaurus build process and scans all Markdown files in the docs directory.
  */
 import type { LoadContext, Plugin } from '@docusaurus/types';
-/**
- * Plugin options interface
- */
-interface PluginOptions {
-    /** Whether to generate the llms.txt file (default: true) */
-    generateLLMsTxt?: boolean;
-    /** Whether to generate the llms-full.txt file (default: true) */
-    generateLLMsFullTxt?: boolean;
-    /** Base directory for documentation files (default: 'docs') */
-    docsDir?: string;
-    /** Array of glob patterns for files to ignore */
-    ignoreFiles?: string[];
-    /** Custom title to use in generated files (defaults to site title) */
-    title?: string;
-    /** Custom description to use in generated files (defaults to site tagline) */
-    description?: string;
-    /** Custom file name for the links file (default: 'llms.txt') */
-    llmsTxtFilename?: string;
-    /** Custom file name for the full content file (default: 'llms-full.txt') */
-    llmsFullTxtFilename?: string;
-    /** Whether to include blog content (default: false) */
-    includeBlog?: boolean;
-    /** Path transformation options for URL construction */
-    pathTransformation?: {
-        /** Path segments to ignore when constructing URLs (will be removed if found) */
-        ignorePaths?: string[];
-        /** Path segments to add when constructing URLs (will be prepended if not already present) */
-        addPaths?: string[];
-    };
-}
+import { PluginOptions } from './types';
 /**
  * A Docusaurus plugin to generate LLM-friendly documentation following
  * the llmtxt.org standard
@@ -47,4 +18,3 @@ interface PluginOptions {
  * @returns Plugin object
  */
 export default function docusaurusPluginLLMs(context: LoadContext, options?: PluginOptions): Plugin<void>;
-export {};