docusaurus-plugin-llms 0.1.2 → 0.1.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +271 -32
- package/lib/generator.d.ts +32 -0
- package/lib/generator.js +212 -0
- package/lib/index.d.ts +2 -31
- package/lib/index.js +39 -335
- package/lib/processor.d.ts +28 -0
- package/lib/processor.js +211 -0
- package/lib/types.d.ts +86 -0
- package/lib/types.js +5 -0
- package/lib/utils.d.ts +53 -0
- package/lib/utils.js +177 -0
- package/package.json +2 -1
- package/src/generator.ts +266 -0
- package/src/index.ts +50 -423
- package/src/processor.ts +236 -0
- package/src/types.ts +113 -0
- package/src/utils.ts +165 -0
package/README.md
CHANGED
@@ -2,6 +2,34 @@
|
|
2
2
|
|
3
3
|
A Docusaurus plugin for generating LLM-friendly documentation following the [llmstxt standard](https://llmstxt.org/).
|
4
4
|
|
5
|
+
## Features
|
6
|
+
|
7
|
+
- ⚡️ Easy integration with Docusaurus
|
8
|
+
- ✅ Zero config required, works out of the box
|
9
|
+
- ⚙️ Highly customizable with multiple options
|
10
|
+
- 📝 Creates `llms.txt` with section links
|
11
|
+
- 📖 Produces `llms-full.txt` with all content in one file
|
12
|
+
- 📋 Document ordering control for custom sequence
|
13
|
+
- 🔄 Path transformation to customize URL construction
|
14
|
+
- 📚 Option to include blog posts
|
15
|
+
- 🧩 Custom LLM files for specific documentation sections
|
16
|
+
- 🧹 Cleans HTML and normalizes content for optimal LLM consumption
|
17
|
+
- 📊 Provides statistics about generated documentation
|
18
|
+
|
19
|
+
## Table of Contents
|
20
|
+
|
21
|
+
- [Installation](#installation)
|
22
|
+
- [Configuration Options](#configuration-options)
|
23
|
+
- [Available Options](#available-options)
|
24
|
+
- [Path Transformation Examples](#path-transformation-examples)
|
25
|
+
- [Document Ordering Examples](#document-ordering-examples)
|
26
|
+
- [Custom LLM Files](#custom-llm-files)
|
27
|
+
- [How It Works](#how-it-works)
|
28
|
+
- [Implementation Details](#implementation-details)
|
29
|
+
- [Testing](#testing)
|
30
|
+
- [Future Enhancements](#future-enhancements)
|
31
|
+
- [License](#license)
|
32
|
+
|
5
33
|
## Installation
|
6
34
|
|
7
35
|
```bash
|
@@ -39,6 +67,13 @@ module.exports = {
|
|
39
67
|
title: 'My Project Documentation',
|
40
68
|
description: 'Complete reference documentation for My Project',
|
41
69
|
includeBlog: true,
|
70
|
+
// Control documentation order
|
71
|
+
includeOrder: [
|
72
|
+
'getting-started/*',
|
73
|
+
'guides/*',
|
74
|
+
'api/*',
|
75
|
+
],
|
76
|
+
includeUnmatchedLast: true,
|
42
77
|
// Path transformation options
|
43
78
|
pathTransformation: {
|
44
79
|
// Paths to ignore when constructing URLs (will be removed if found)
|
@@ -46,6 +81,23 @@ module.exports = {
|
|
46
81
|
// Paths to add when constructing URLs (will be prepended if not already present)
|
47
82
|
addPaths: ['api'],
|
48
83
|
},
|
84
|
+
// Custom LLM files for specific documentation sections
|
85
|
+
customLLMFiles: [
|
86
|
+
{
|
87
|
+
filename: 'llms-python.txt',
|
88
|
+
includePatterns: ['api/python/**/*.md', 'guides/python/*.md'],
|
89
|
+
fullContent: true,
|
90
|
+
title: 'Python API Documentation',
|
91
|
+
description: 'Complete reference for Python API'
|
92
|
+
},
|
93
|
+
{
|
94
|
+
filename: 'llms-tutorials.txt',
|
95
|
+
includePatterns: ['tutorials/**/*.md'],
|
96
|
+
fullContent: false,
|
97
|
+
title: 'Tutorial Documentation',
|
98
|
+
description: 'All tutorials in a single file'
|
99
|
+
}
|
100
|
+
],
|
49
101
|
},
|
50
102
|
],
|
51
103
|
// ... your other plugins
|
@@ -55,20 +107,24 @@ module.exports = {
|
|
55
107
|
|
56
108
|
### Available Options
|
57
109
|
|
58
|
-
| Option
|
59
|
-
|
60
|
-
| `
|
61
|
-
| `
|
62
|
-
| `
|
63
|
-
| `
|
64
|
-
| `
|
65
|
-
| `
|
66
|
-
| `
|
67
|
-
| `
|
68
|
-
| `
|
69
|
-
| `
|
70
|
-
| `pathTransformation.
|
71
|
-
| `pathTransformation.
|
110
|
+
| Option | Type | Default | Description |
|
111
|
+
|----------------------------------|----------|-------------------|---------------------------------------------------------------|
|
112
|
+
| `description` | string | Site tagline | Custom description to use in generated files |
|
113
|
+
| `docsDir` | string | `'docs'` | Base directory for documentation files |
|
114
|
+
| `generateLLMsFullTxt` | boolean | `true` | Whether to generate the full content file |
|
115
|
+
| `generateLLMsTxt` | boolean | `true` | Whether to generate the links file |
|
116
|
+
| `ignoreFiles` | string[] | `[]` | Array of glob patterns for files to ignore |
|
117
|
+
| `includeBlog` | boolean | `false` | Whether to include blog content |
|
118
|
+
| `includeOrder` | string[] | `[]` | Array of glob patterns for files to process in specific order |
|
119
|
+
| `includeUnmatchedLast` | boolean | `true` | Whether to include unmatched files at the end |
|
120
|
+
| `llmsFullTxtFilename` | string | `'llms-full.txt'` | Custom filename for the full content file |
|
121
|
+
| `llmsTxtFilename` | string | `'llms.txt'` | Custom filename for the links file |
|
122
|
+
| `pathTransformation.addPaths` | string[] | `[]` | Path segments to add when constructing URLs |
|
123
|
+
| `pathTransformation.ignorePaths` | string[] | `[]` | Path segments to ignore when constructing URLs |
|
124
|
+
| `pathTransformation` | object | `undefined` | Path transformation options for URL construction |
|
125
|
+
| `title` | string | Site title | Custom title to use in generated files |
|
126
|
+
| `version` | string | `undefined` | Global version to include in all generated files |
|
127
|
+
| `customLLMFiles` | array | `[]` | Array of custom LLM file configurations |
|
72
128
|
|
73
129
|
### Path Transformation Examples
|
74
130
|
|
@@ -101,26 +157,207 @@ File path: `/content/docs/manual/decorators.md` → URL: `https://example.com/ap
|
|
101
157
|
|
102
158
|
The configuration supports multiple path segments in both arrays.
|
103
159
|
|
160
|
+
### Document Ordering Examples
|
161
|
+
|
162
|
+
The document ordering feature allows you to control the sequence in which files appear in the generated output:
|
163
|
+
|
164
|
+
**Example 1**: Basic Section Ordering
|
165
|
+
```js
|
166
|
+
includeOrder: [
|
167
|
+
'getting-started/*',
|
168
|
+
'guides/*',
|
169
|
+
'api/*',
|
170
|
+
'advanced/*'
|
171
|
+
]
|
172
|
+
```
|
173
|
+
Result: Files will appear in the generated output following this section order.
|
174
|
+
|
175
|
+
**Example 2**: Strict Inclusion List
|
176
|
+
```js
|
177
|
+
includeOrder: [
|
178
|
+
'public-docs/**/*.md'
|
179
|
+
],
|
180
|
+
includeUnmatchedLast: false
|
181
|
+
```
|
182
|
+
Result: Only files matching 'public-docs/**/*.md' are included, all others are excluded.
|
183
|
+
|
184
|
+
**Example 3**: Detailed Ordering with Specific Files First
|
185
|
+
```js
|
186
|
+
includeOrder: [
|
187
|
+
'getting-started/installation.md',
|
188
|
+
'getting-started/quick-start.md',
|
189
|
+
'getting-started/*.md',
|
190
|
+
'api/core/*.md',
|
191
|
+
'api/plugins/*.md',
|
192
|
+
'api/**/*.md'
|
193
|
+
]
|
194
|
+
```
|
195
|
+
Result: Installation and quick-start guides appear first, followed by other getting-started files, then API documentation in a specific order.
|
196
|
+
|
197
|
+
### Custom LLM Files
|
198
|
+
|
199
|
+
In addition to the standard `llms.txt` and `llms-full.txt` files, you can generate custom LLM-friendly files for different sections of your documentation with the `customLLMFiles` option:
|
200
|
+
|
201
|
+
```js
|
202
|
+
customLLMFiles: [
|
203
|
+
{
|
204
|
+
filename: 'llms-python.txt',
|
205
|
+
includePatterns: ['api/python/**/*.md', 'guides/python/*.md'],
|
206
|
+
fullContent: true,
|
207
|
+
title: 'Python API Documentation',
|
208
|
+
description: 'Complete reference for Python API'
|
209
|
+
},
|
210
|
+
{
|
211
|
+
filename: 'llms-tutorials.txt',
|
212
|
+
includePatterns: ['tutorials/**/*.md'],
|
213
|
+
fullContent: false,
|
214
|
+
title: 'Tutorial Documentation',
|
215
|
+
description: 'All tutorials in a single file'
|
216
|
+
}
|
217
|
+
]
|
218
|
+
```
|
219
|
+
|
220
|
+
#### Custom LLM File Configuration
|
221
|
+
|
222
|
+
Each custom LLM file is defined by an object with the following properties:
|
223
|
+
|
224
|
+
| Option | Type | Required | Description |
|
225
|
+
|-----------------------|----------|----------|----------------------------------------------|
|
226
|
+
| `filename` | string | Yes | Name of the output file (e.g., 'llms-python.txt') |
|
227
|
+
| `includePatterns` | string[] | Yes | Glob patterns for files to include |
|
228
|
+
| `fullContent` | boolean | Yes | `true` for full content like llms-full.txt, `false` for links only like llms.txt |
|
229
|
+
| `title` | string | No | Custom title for this file (defaults to site title) |
|
230
|
+
| `description` | string | No | Custom description for this file (defaults to site description) |
|
231
|
+
| `ignorePatterns` | string[] | No | Additional patterns to exclude (combined with global ignoreFiles) |
|
232
|
+
| `orderPatterns` | string[] | No | Order patterns for controlling file ordering (similar to includeOrder) |
|
233
|
+
| `includeUnmatchedLast`| boolean | No | Whether to include unmatched files last (default: false) |
|
234
|
+
| `version` | string | No | Version information for this LLM file (overrides global version) |
|
235
|
+
|
236
|
+
#### Use Cases
|
237
|
+
|
238
|
+
##### Language-Specific Documentation
|
239
|
+
|
240
|
+
Create separate files for different programming languages:
|
241
|
+
|
242
|
+
```js
|
243
|
+
customLLMFiles: [
|
244
|
+
{
|
245
|
+
filename: 'llms-python.txt',
|
246
|
+
includePatterns: ['api/python/**/*.md', 'guides/python/*.md'],
|
247
|
+
fullContent: true,
|
248
|
+
title: 'Python API Documentation'
|
249
|
+
},
|
250
|
+
{
|
251
|
+
filename: 'llms-javascript.txt',
|
252
|
+
includePatterns: ['api/javascript/**/*.md', 'guides/javascript/*.md'],
|
253
|
+
fullContent: true,
|
254
|
+
title: 'JavaScript API Documentation'
|
255
|
+
}
|
256
|
+
]
|
257
|
+
```
|
258
|
+
|
259
|
+
##### Content Type Separation
|
260
|
+
|
261
|
+
Separate tutorials from API reference:
|
262
|
+
|
263
|
+
```js
|
264
|
+
customLLMFiles: [
|
265
|
+
{
|
266
|
+
filename: 'llms-tutorials.txt',
|
267
|
+
includePatterns: ['tutorials/**/*.md', 'guides/**/*.md'],
|
268
|
+
fullContent: true,
|
269
|
+
title: 'Tutorials and Guides'
|
270
|
+
},
|
271
|
+
{
|
272
|
+
filename: 'llms-api.txt',
|
273
|
+
includePatterns: ['api/**/*.md', 'reference/**/*.md'],
|
274
|
+
fullContent: true,
|
275
|
+
title: 'API Reference'
|
276
|
+
}
|
277
|
+
]
|
278
|
+
```
|
279
|
+
|
280
|
+
##### Beginner-Friendly Documentation
|
281
|
+
|
282
|
+
Create a beginner-focused file with carefully ordered content:
|
283
|
+
|
284
|
+
```js
|
285
|
+
customLLMFiles: [
|
286
|
+
{
|
287
|
+
filename: 'llms-getting-started.txt',
|
288
|
+
includePatterns: ['**/*.md'],
|
289
|
+
ignorePatterns: ['advanced/**/*.md', 'internal/**/*.md'],
|
290
|
+
orderPatterns: [
|
291
|
+
'introduction.md',
|
292
|
+
'getting-started/*.md',
|
293
|
+
'tutorials/basic/*.md',
|
294
|
+
'examples/simple/*.md'
|
295
|
+
],
|
296
|
+
fullContent: true,
|
297
|
+
title: 'Getting Started Guide',
|
298
|
+
description: 'Beginner-friendly documentation with essential concepts'
|
299
|
+
}
|
300
|
+
]
|
301
|
+
```
|
302
|
+
|
303
|
+
##### Versioned Documentation
|
304
|
+
|
305
|
+
Include version information in your documentation files:
|
306
|
+
|
307
|
+
```js
|
308
|
+
plugins: [
|
309
|
+
[
|
310
|
+
'docusaurus-plugin-llms',
|
311
|
+
{
|
312
|
+
// Global version applies to all files
|
313
|
+
version: '2.0.0',
|
314
|
+
|
315
|
+
// Custom LLM files with specific versions
|
316
|
+
customLLMFiles: [
|
317
|
+
{
|
318
|
+
filename: 'api-reference.txt',
|
319
|
+
title: 'API Reference Documentation',
|
320
|
+
description: 'Complete API reference for developers',
|
321
|
+
includePatterns: ['**/api/**/*.md', '**/reference/**/*.md'],
|
322
|
+
fullContent: true,
|
323
|
+
version: '1.0.0' // Overrides global version
|
324
|
+
},
|
325
|
+
{
|
326
|
+
filename: 'tutorials.txt',
|
327
|
+
title: 'Tutorials and Guides',
|
328
|
+
description: 'Step-by-step tutorials and guides',
|
329
|
+
includePatterns: ['**/tutorials/**/*.md', '**/guides/**/*.md'],
|
330
|
+
fullContent: true,
|
331
|
+
version: '0.9.5-beta' // Overrides global version
|
332
|
+
}
|
333
|
+
]
|
334
|
+
}
|
335
|
+
],
|
336
|
+
]
|
337
|
+
```
|
338
|
+
|
339
|
+
The generated files will include the version information under the description:
|
340
|
+
|
341
|
+
```
|
342
|
+
# API Reference Documentation
|
343
|
+
|
344
|
+
> Complete API reference for developers
|
345
|
+
|
346
|
+
Version: 1.0.0
|
347
|
+
|
348
|
+
This file contains all documentation content in a single document following the llmtxt.org standard.
|
349
|
+
```
|
350
|
+
|
104
351
|
## How It Works
|
105
352
|
|
106
353
|
This plugin automatically generates the following files during the build process:
|
107
354
|
|
108
355
|
- **llms.txt**: Contains links to all sections of your documentation
|
109
356
|
- **llms-full.txt**: Contains all documentation content in a single file
|
357
|
+
- **Custom LLM files**: Additional files based on your custom configurations
|
110
358
|
|
111
359
|
These files follow the [llmstxt standard](https://llmstxt.org/), making your documentation optimized for use with Large Language Models (LLMs).
|
112
360
|
|
113
|
-
## Features
|
114
|
-
|
115
|
-
- ⚡️ Easy integration with Docusaurus
|
116
|
-
- ✅ Zero config required, works out of the box
|
117
|
-
- ⚙️ Highly customizable with multiple options
|
118
|
-
- 📝 Creates `llms.txt` with section links
|
119
|
-
- 📖 Produces `llms-full.txt` with all content in one file
|
120
|
-
- 🧹 Cleans HTML and normalizes content for optimal LLM consumption
|
121
|
-
- 📊 Provides statistics about generated documentation
|
122
|
-
- 📚 Option to include blog posts
|
123
|
-
- 🔄 Path transformation to customize URL construction
|
124
361
|
|
125
362
|
## Implementation Details
|
126
363
|
|
@@ -128,12 +365,14 @@ The plugin:
|
|
128
365
|
|
129
366
|
1. Scans your `docs` directory recursively for all Markdown files
|
130
367
|
2. Optionally includes blog content
|
131
|
-
3.
|
132
|
-
4.
|
133
|
-
5.
|
134
|
-
6.
|
135
|
-
7.
|
136
|
-
8.
|
368
|
+
3. Orders documents according to specified glob patterns (if provided)
|
369
|
+
4. Extracts metadata, titles, and content from each file
|
370
|
+
5. Creates proper URL links to each document section
|
371
|
+
6. Applies path transformations according to configuration (removing or adding path segments)
|
372
|
+
7. Generates a table of contents in `llms.txt`
|
373
|
+
8. Combines all documentation content in `llms-full.txt`
|
374
|
+
9. Creates custom LLM files based on specified configurations
|
375
|
+
10. Provides statistics about the generated documentation
|
137
376
|
|
138
377
|
## Testing
|
139
378
|
|
@@ -166,5 +405,5 @@ Planned features for future versions:
|
|
166
405
|
- Specific content tags for LLM-only sections
|
167
406
|
|
168
407
|
## License
|
408
|
+
This project is licensed under the MIT License.
|
169
409
|
|
170
|
-
MIT
|
@@ -0,0 +1,32 @@
|
|
1
|
+
/**
|
2
|
+
* LLM file generation functions for the docusaurus-plugin-llms plugin
|
3
|
+
*/
|
4
|
+
import { DocInfo, PluginContext } from './types';
|
5
|
+
/**
|
6
|
+
* Generate an LLM-friendly file
|
7
|
+
* @param docs - Processed document information
|
8
|
+
* @param outputPath - Path to write the output file
|
9
|
+
* @param fileTitle - Title for the file
|
10
|
+
* @param fileDescription - Description for the file
|
11
|
+
* @param includeFullContent - Whether to include full content or just links
|
12
|
+
* @param version - Version of the file
|
13
|
+
*/
|
14
|
+
export declare function generateLLMFile(docs: DocInfo[], outputPath: string, fileTitle: string, fileDescription: string, includeFullContent: boolean, version?: string): Promise<void>;
|
15
|
+
/**
|
16
|
+
* Generate standard LLM files (llms.txt and llms-full.txt)
|
17
|
+
* @param context - Plugin context
|
18
|
+
* @param allDocFiles - Array of all document files
|
19
|
+
*/
|
20
|
+
export declare function generateStandardLLMFiles(context: PluginContext, allDocFiles: string[]): Promise<void>;
|
21
|
+
/**
|
22
|
+
* Generate custom LLM files based on configuration
|
23
|
+
* @param context - Plugin context
|
24
|
+
* @param allDocFiles - Array of all document files
|
25
|
+
*/
|
26
|
+
export declare function generateCustomLLMFiles(context: PluginContext, allDocFiles: string[]): Promise<void>;
|
27
|
+
/**
|
28
|
+
* Collect all document files from docs directory and optionally blog
|
29
|
+
* @param context - Plugin context
|
30
|
+
* @returns Array of file paths
|
31
|
+
*/
|
32
|
+
export declare function collectDocFiles(context: PluginContext): Promise<string[]>;
|
package/lib/generator.js
ADDED
@@ -0,0 +1,212 @@
|
|
1
|
+
"use strict";
|
2
|
+
/**
|
3
|
+
* LLM file generation functions for the docusaurus-plugin-llms plugin
|
4
|
+
*/
|
5
|
+
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
6
|
+
if (k2 === undefined) k2 = k;
|
7
|
+
var desc = Object.getOwnPropertyDescriptor(m, k);
|
8
|
+
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
9
|
+
desc = { enumerable: true, get: function() { return m[k]; } };
|
10
|
+
}
|
11
|
+
Object.defineProperty(o, k2, desc);
|
12
|
+
}) : (function(o, m, k, k2) {
|
13
|
+
if (k2 === undefined) k2 = k;
|
14
|
+
o[k2] = m[k];
|
15
|
+
}));
|
16
|
+
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
17
|
+
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
18
|
+
}) : function(o, v) {
|
19
|
+
o["default"] = v;
|
20
|
+
});
|
21
|
+
var __importStar = (this && this.__importStar) || (function () {
|
22
|
+
var ownKeys = function(o) {
|
23
|
+
ownKeys = Object.getOwnPropertyNames || function (o) {
|
24
|
+
var ar = [];
|
25
|
+
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
|
26
|
+
return ar;
|
27
|
+
};
|
28
|
+
return ownKeys(o);
|
29
|
+
};
|
30
|
+
return function (mod) {
|
31
|
+
if (mod && mod.__esModule) return mod;
|
32
|
+
var result = {};
|
33
|
+
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
|
34
|
+
__setModuleDefault(result, mod);
|
35
|
+
return result;
|
36
|
+
};
|
37
|
+
})();
|
38
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
39
|
+
exports.generateLLMFile = generateLLMFile;
|
40
|
+
exports.generateStandardLLMFiles = generateStandardLLMFiles;
|
41
|
+
exports.generateCustomLLMFiles = generateCustomLLMFiles;
|
42
|
+
exports.collectDocFiles = collectDocFiles;
|
43
|
+
const path = __importStar(require("path"));
|
44
|
+
const fs = __importStar(require("fs/promises"));
|
45
|
+
const utils_1 = require("./utils");
|
46
|
+
const processor_1 = require("./processor");
|
47
|
+
/**
|
48
|
+
* Clean a description for use in a TOC item
|
49
|
+
* @param description - The original description
|
50
|
+
* @returns Cleaned description suitable for TOC
|
51
|
+
*/
|
52
|
+
function cleanDescriptionForToc(description) {
|
53
|
+
if (!description)
|
54
|
+
return '';
|
55
|
+
// Get just the first line for TOC display
|
56
|
+
const firstLine = description.split('\n')[0];
|
57
|
+
// Remove heading markers only at the beginning of the line
|
58
|
+
// Be careful to only remove actual heading markers (# followed by space at beginning)
|
59
|
+
// and not hashtag symbols that are part of the content (inline hashtags)
|
60
|
+
const cleaned = firstLine.replace(/^(#+)\s+/g, '');
|
61
|
+
// Truncate if too long (150 characters max with ellipsis)
|
62
|
+
return cleaned.length > 150 ? cleaned.substring(0, 147) + '...' : cleaned;
|
63
|
+
}
|
64
|
+
/**
|
65
|
+
* Generate an LLM-friendly file
|
66
|
+
* @param docs - Processed document information
|
67
|
+
* @param outputPath - Path to write the output file
|
68
|
+
* @param fileTitle - Title for the file
|
69
|
+
* @param fileDescription - Description for the file
|
70
|
+
* @param includeFullContent - Whether to include full content or just links
|
71
|
+
* @param version - Version of the file
|
72
|
+
*/
|
73
|
+
async function generateLLMFile(docs, outputPath, fileTitle, fileDescription, includeFullContent, version) {
|
74
|
+
console.log(`Generating file: ${outputPath}, version: ${version || 'undefined'}`);
|
75
|
+
const versionInfo = version ? `\n\nVersion: ${version}` : '';
|
76
|
+
if (includeFullContent) {
|
77
|
+
// Generate full content file
|
78
|
+
const fullContentSections = docs.map(doc => {
|
79
|
+
return `## ${doc.title}
|
80
|
+
|
81
|
+
${doc.content}`;
|
82
|
+
});
|
83
|
+
const llmFileContent = `# ${fileTitle}
|
84
|
+
|
85
|
+
> ${fileDescription}${versionInfo}
|
86
|
+
|
87
|
+
This file contains all documentation content in a single document following the llmtxt.org standard.
|
88
|
+
|
89
|
+
${fullContentSections.join('\n\n---\n\n')}
|
90
|
+
`;
|
91
|
+
await (0, utils_1.writeFile)(outputPath, llmFileContent);
|
92
|
+
}
|
93
|
+
else {
|
94
|
+
// Generate links-only file
|
95
|
+
const tocItems = docs.map(doc => {
|
96
|
+
// Clean and format the description for TOC
|
97
|
+
const cleanedDescription = cleanDescriptionForToc(doc.description);
|
98
|
+
return `- [${doc.title}](${doc.url})${cleanedDescription ? `: ${cleanedDescription}` : ''}`;
|
99
|
+
});
|
100
|
+
const llmFileContent = `# ${fileTitle}
|
101
|
+
|
102
|
+
> ${fileDescription}${versionInfo}
|
103
|
+
|
104
|
+
This file contains links to documentation sections following the llmstxt.org standard.
|
105
|
+
|
106
|
+
## Table of Contents
|
107
|
+
|
108
|
+
${tocItems.join('\n')}
|
109
|
+
`;
|
110
|
+
await (0, utils_1.writeFile)(outputPath, llmFileContent);
|
111
|
+
}
|
112
|
+
console.log(`Generated: ${outputPath}`);
|
113
|
+
}
|
114
|
+
/**
|
115
|
+
* Generate standard LLM files (llms.txt and llms-full.txt)
|
116
|
+
* @param context - Plugin context
|
117
|
+
* @param allDocFiles - Array of all document files
|
118
|
+
*/
|
119
|
+
async function generateStandardLLMFiles(context, allDocFiles) {
|
120
|
+
const { outDir, docTitle, docDescription, options } = context;
|
121
|
+
const { generateLLMsTxt, generateLLMsFullTxt, llmsTxtFilename = 'llms.txt', llmsFullTxtFilename = 'llms-full.txt', includeOrder = [], includeUnmatchedLast = true, version } = options;
|
122
|
+
if (!generateLLMsTxt && !generateLLMsFullTxt) {
|
123
|
+
return;
|
124
|
+
}
|
125
|
+
// Process files for the standard outputs
|
126
|
+
const processedDocs = await (0, processor_1.processFilesWithPatterns)(context, allDocFiles, [], // No specific include patterns - include all
|
127
|
+
[], // No additional ignore patterns beyond global ignoreFiles
|
128
|
+
includeOrder, includeUnmatchedLast);
|
129
|
+
console.log(`Processed ${processedDocs.length} documentation files for standard LLM files`);
|
130
|
+
// Generate llms.txt
|
131
|
+
if (generateLLMsTxt) {
|
132
|
+
const llmsTxtPath = path.join(outDir, llmsTxtFilename);
|
133
|
+
await generateLLMFile(processedDocs, llmsTxtPath, docTitle, docDescription, false, // links only
|
134
|
+
version);
|
135
|
+
}
|
136
|
+
// Generate llms-full.txt
|
137
|
+
if (generateLLMsFullTxt) {
|
138
|
+
const llmsFullTxtPath = path.join(outDir, llmsFullTxtFilename);
|
139
|
+
await generateLLMFile(processedDocs, llmsFullTxtPath, docTitle, docDescription, true, // full content
|
140
|
+
version);
|
141
|
+
}
|
142
|
+
}
|
143
|
+
/**
|
144
|
+
* Generate custom LLM files based on configuration
|
145
|
+
* @param context - Plugin context
|
146
|
+
* @param allDocFiles - Array of all document files
|
147
|
+
*/
|
148
|
+
async function generateCustomLLMFiles(context, allDocFiles) {
|
149
|
+
const { outDir, docTitle, docDescription, options } = context;
|
150
|
+
const { customLLMFiles = [], ignoreFiles = [] } = options;
|
151
|
+
if (customLLMFiles.length === 0) {
|
152
|
+
return;
|
153
|
+
}
|
154
|
+
console.log(`Generating ${customLLMFiles.length} custom LLM files...`);
|
155
|
+
for (const customFile of customLLMFiles) {
|
156
|
+
console.log(`Processing custom file: ${customFile.filename}, version: ${customFile.version || 'undefined'}`);
|
157
|
+
// Combine global ignores with custom ignores
|
158
|
+
const combinedIgnores = [...ignoreFiles];
|
159
|
+
if (customFile.ignorePatterns) {
|
160
|
+
combinedIgnores.push(...customFile.ignorePatterns);
|
161
|
+
}
|
162
|
+
// Process files according to the custom configuration
|
163
|
+
const customDocs = await (0, processor_1.processFilesWithPatterns)(context, allDocFiles, customFile.includePatterns, combinedIgnores, customFile.orderPatterns || [], customFile.includeUnmatchedLast ?? false);
|
164
|
+
if (customDocs.length > 0) {
|
165
|
+
// Use custom title/description or fall back to defaults
|
166
|
+
const customTitle = customFile.title || docTitle;
|
167
|
+
const customDescription = customFile.description || docDescription;
|
168
|
+
// Generate the custom LLM file
|
169
|
+
const customFilePath = path.join(outDir, customFile.filename);
|
170
|
+
await generateLLMFile(customDocs, customFilePath, customTitle, customDescription, customFile.fullContent, customFile.version);
|
171
|
+
console.log(`Generated custom LLM file: ${customFile.filename} with ${customDocs.length} documents`);
|
172
|
+
}
|
173
|
+
else {
|
174
|
+
console.warn(`No matching documents found for custom LLM file: ${customFile.filename}`);
|
175
|
+
}
|
176
|
+
}
|
177
|
+
}
|
178
|
+
/**
|
179
|
+
* Collect all document files from docs directory and optionally blog
|
180
|
+
* @param context - Plugin context
|
181
|
+
* @returns Array of file paths
|
182
|
+
*/
|
183
|
+
async function collectDocFiles(context) {
|
184
|
+
const { siteDir, docsDir, options } = context;
|
185
|
+
const { ignoreFiles = [], includeBlog = false } = options;
|
186
|
+
const allDocFiles = [];
|
187
|
+
// Process docs directory
|
188
|
+
const fullDocsDir = path.join(siteDir, docsDir);
|
189
|
+
try {
|
190
|
+
await fs.access(fullDocsDir);
|
191
|
+
// Collect all markdown files from docs directory
|
192
|
+
const docFiles = await (0, utils_1.readMarkdownFiles)(fullDocsDir, siteDir, ignoreFiles);
|
193
|
+
allDocFiles.push(...docFiles);
|
194
|
+
}
|
195
|
+
catch (err) {
|
196
|
+
console.warn(`Docs directory not found: ${fullDocsDir}`);
|
197
|
+
}
|
198
|
+
// Process blog if enabled
|
199
|
+
if (includeBlog) {
|
200
|
+
const blogDir = path.join(siteDir, 'blog');
|
201
|
+
try {
|
202
|
+
await fs.access(blogDir);
|
203
|
+
// Collect all markdown files from blog directory
|
204
|
+
const blogFiles = await (0, utils_1.readMarkdownFiles)(blogDir, siteDir, ignoreFiles);
|
205
|
+
allDocFiles.push(...blogFiles);
|
206
|
+
}
|
207
|
+
catch (err) {
|
208
|
+
console.warn(`Blog directory not found: ${blogDir}`);
|
209
|
+
}
|
210
|
+
}
|
211
|
+
return allDocFiles;
|
212
|
+
}
|
package/lib/index.d.ts
CHANGED
@@ -8,36 +8,7 @@
|
|
8
8
|
* The plugin runs during the Docusaurus build process and scans all Markdown files in the docs directory.
|
9
9
|
*/
|
10
10
|
import type { LoadContext, Plugin } from '@docusaurus/types';
|
11
|
-
|
12
|
-
* Plugin options interface
|
13
|
-
*/
|
14
|
-
interface PluginOptions {
|
15
|
-
/** Whether to generate the llms.txt file (default: true) */
|
16
|
-
generateLLMsTxt?: boolean;
|
17
|
-
/** Whether to generate the llms-full.txt file (default: true) */
|
18
|
-
generateLLMsFullTxt?: boolean;
|
19
|
-
/** Base directory for documentation files (default: 'docs') */
|
20
|
-
docsDir?: string;
|
21
|
-
/** Array of glob patterns for files to ignore */
|
22
|
-
ignoreFiles?: string[];
|
23
|
-
/** Custom title to use in generated files (defaults to site title) */
|
24
|
-
title?: string;
|
25
|
-
/** Custom description to use in generated files (defaults to site tagline) */
|
26
|
-
description?: string;
|
27
|
-
/** Custom file name for the links file (default: 'llms.txt') */
|
28
|
-
llmsTxtFilename?: string;
|
29
|
-
/** Custom file name for the full content file (default: 'llms-full.txt') */
|
30
|
-
llmsFullTxtFilename?: string;
|
31
|
-
/** Whether to include blog content (default: false) */
|
32
|
-
includeBlog?: boolean;
|
33
|
-
/** Path transformation options for URL construction */
|
34
|
-
pathTransformation?: {
|
35
|
-
/** Path segments to ignore when constructing URLs (will be removed if found) */
|
36
|
-
ignorePaths?: string[];
|
37
|
-
/** Path segments to add when constructing URLs (will be prepended if not already present) */
|
38
|
-
addPaths?: string[];
|
39
|
-
};
|
40
|
-
}
|
11
|
+
import { PluginOptions } from './types';
|
41
12
|
/**
|
42
13
|
* A Docusaurus plugin to generate LLM-friendly documentation following
|
43
14
|
* the llmtxt.org standard
|
@@ -47,4 +18,4 @@ interface PluginOptions {
|
|
47
18
|
* @returns Plugin object
|
48
19
|
*/
|
49
20
|
export default function docusaurusPluginLLMs(context: LoadContext, options?: PluginOptions): Plugin<void>;
|
50
|
-
export {};
|
21
|
+
export type { PluginOptions };
|