docusaurus-plugin-llms 0.1.3 → 0.1.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +30 -15
- package/lib/generator.js +2 -2
- package/lib/index.d.ts +3 -2
- package/lib/index.js +2 -2
- package/lib/processor.d.ts +1 -1
- package/lib/processor.js +7 -1
- package/lib/types.d.ts +86 -0
- package/lib/types.js +5 -0
- package/package.json +3 -2
- package/src/generator.ts +2 -2
- package/src/index.ts +5 -3
- package/src/processor.ts +9 -2
package/README.md
CHANGED
@@ -2,6 +2,34 @@
|
|
2
2
|
|
3
3
|
A Docusaurus plugin for generating LLM-friendly documentation following the [llmstxt standard](https://llmstxt.org/).
|
4
4
|
|
5
|
+
## Features
|
6
|
+
|
7
|
+
- ⚡️ Easy integration with Docusaurus
|
8
|
+
- ✅ Zero config required, works out of the box
|
9
|
+
- ⚙️ Highly customizable with multiple options
|
10
|
+
- 📝 Creates `llms.txt` with section links
|
11
|
+
- 📖 Produces `llms-full.txt` with all content in one file
|
12
|
+
- 📋 Document ordering control for custom sequence
|
13
|
+
- 🔄 Path transformation to customize URL construction
|
14
|
+
- 📚 Option to include blog posts
|
15
|
+
- 🧩 Custom LLM files for specific documentation sections
|
16
|
+
- 🧹 Cleans HTML and normalizes content for optimal LLM consumption
|
17
|
+
- 📊 Provides statistics about generated documentation
|
18
|
+
|
19
|
+
## Table of Contents
|
20
|
+
|
21
|
+
- [Installation](#installation)
|
22
|
+
- [Configuration Options](#configuration-options)
|
23
|
+
- [Available Options](#available-options)
|
24
|
+
- [Path Transformation Examples](#path-transformation-examples)
|
25
|
+
- [Document Ordering Examples](#document-ordering-examples)
|
26
|
+
- [Custom LLM Files](#custom-llm-files)
|
27
|
+
- [How It Works](#how-it-works)
|
28
|
+
- [Implementation Details](#implementation-details)
|
29
|
+
- [Testing](#testing)
|
30
|
+
- [Future Enhancements](#future-enhancements)
|
31
|
+
- [License](#license)
|
32
|
+
|
5
33
|
## Installation
|
6
34
|
|
7
35
|
```bash
|
@@ -317,7 +345,7 @@ The generated files will include the version information under the description:
|
|
317
345
|
|
318
346
|
Version: 1.0.0
|
319
347
|
|
320
|
-
This file contains all documentation content in a single document following the
|
348
|
+
This file contains all documentation content in a single document following the llmstxt.org standard.
|
321
349
|
```
|
322
350
|
|
323
351
|
## How It Works
|
@@ -330,19 +358,6 @@ This plugin automatically generates the following files during the build process
|
|
330
358
|
|
331
359
|
These files follow the [llmstxt standard](https://llmstxt.org/), making your documentation optimized for use with Large Language Models (LLMs).
|
332
360
|
|
333
|
-
## Features
|
334
|
-
|
335
|
-
- ⚡️ Easy integration with Docusaurus
|
336
|
-
- ✅ Zero config required, works out of the box
|
337
|
-
- ⚙️ Highly customizable with multiple options
|
338
|
-
- 📝 Creates `llms.txt` with section links
|
339
|
-
- 📖 Produces `llms-full.txt` with all content in one file
|
340
|
-
- 📋 Document ordering control for custom sequence
|
341
|
-
- 🔄 Path transformation to customize URL construction
|
342
|
-
- 📚 Option to include blog posts
|
343
|
-
- 🧩 Custom LLM files for specific documentation sections
|
344
|
-
- 🧹 Cleans HTML and normalizes content for optimal LLM consumption
|
345
|
-
- 📊 Provides statistics about generated documentation
|
346
361
|
|
347
362
|
## Implementation Details
|
348
363
|
|
@@ -390,5 +405,5 @@ Planned features for future versions:
|
|
390
405
|
- Specific content tags for LLM-only sections
|
391
406
|
|
392
407
|
## License
|
408
|
+
This project is licensed under the MIT License.
|
393
409
|
|
394
|
-
MIT
|
package/lib/generator.js
CHANGED
@@ -84,7 +84,7 @@ ${doc.content}`;
|
|
84
84
|
|
85
85
|
> ${fileDescription}${versionInfo}
|
86
86
|
|
87
|
-
This file contains all documentation content in a single document following the
|
87
|
+
This file contains all documentation content in a single document following the llmstxt.org standard.
|
88
88
|
|
89
89
|
${fullContentSections.join('\n\n---\n\n')}
|
90
90
|
`;
|
@@ -101,7 +101,7 @@ ${fullContentSections.join('\n\n---\n\n')}
|
|
101
101
|
|
102
102
|
> ${fileDescription}${versionInfo}
|
103
103
|
|
104
|
-
This file contains links to documentation sections following the
|
104
|
+
This file contains links to documentation sections following the llmstxt.org standard.
|
105
105
|
|
106
106
|
## Table of Contents
|
107
107
|
|
package/lib/index.d.ts
CHANGED
@@ -1,5 +1,5 @@
|
|
1
1
|
/**
|
2
|
-
* @fileoverview Docusaurus plugin that generates LLM-friendly documentation following the
|
2
|
+
* @fileoverview Docusaurus plugin that generates LLM-friendly documentation following the llmstxt.org standard.
|
3
3
|
*
|
4
4
|
* This plugin creates two files:
|
5
5
|
* - llms.txt: Contains links to all sections of documentation
|
@@ -11,10 +11,11 @@ import type { LoadContext, Plugin } from '@docusaurus/types';
|
|
11
11
|
import { PluginOptions } from './types';
|
12
12
|
/**
|
13
13
|
* A Docusaurus plugin to generate LLM-friendly documentation following
|
14
|
-
* the
|
14
|
+
* the llmstxt.org standard
|
15
15
|
*
|
16
16
|
* @param context - Docusaurus context
|
17
17
|
* @param options - Plugin options
|
18
18
|
* @returns Plugin object
|
19
19
|
*/
|
20
20
|
export default function docusaurusPluginLLMs(context: LoadContext, options?: PluginOptions): Plugin<void>;
|
21
|
+
export type { PluginOptions };
|
package/lib/index.js
CHANGED
@@ -1,6 +1,6 @@
|
|
1
1
|
"use strict";
|
2
2
|
/**
|
3
|
-
* @fileoverview Docusaurus plugin that generates LLM-friendly documentation following the
|
3
|
+
* @fileoverview Docusaurus plugin that generates LLM-friendly documentation following the llmstxt.org standard.
|
4
4
|
*
|
5
5
|
* This plugin creates two files:
|
6
6
|
* - llms.txt: Contains links to all sections of documentation
|
@@ -13,7 +13,7 @@ exports.default = docusaurusPluginLLMs;
|
|
13
13
|
const generator_1 = require("./generator");
|
14
14
|
/**
|
15
15
|
* A Docusaurus plugin to generate LLM-friendly documentation following
|
16
|
-
* the
|
16
|
+
* the llmstxt.org standard
|
17
17
|
*
|
18
18
|
* @param context - Docusaurus context
|
19
19
|
* @param options - Plugin options
|
package/lib/processor.d.ts
CHANGED
@@ -14,7 +14,7 @@ import { DocInfo, PluginContext } from './types';
|
|
14
14
|
export declare function processMarkdownFile(filePath: string, baseDir: string, siteUrl: string, pathPrefix?: string, pathTransformation?: {
|
15
15
|
ignorePaths?: string[];
|
16
16
|
addPaths?: string[];
|
17
|
-
}): Promise<DocInfo>;
|
17
|
+
}): Promise<DocInfo | null>;
|
18
18
|
/**
|
19
19
|
* Process files based on include patterns, ignore patterns, and ordering
|
20
20
|
* @param context - Plugin context
|
package/lib/processor.js
CHANGED
@@ -57,6 +57,10 @@ const utils_1 = require("./utils");
|
|
57
57
|
async function processMarkdownFile(filePath, baseDir, siteUrl, pathPrefix = 'docs', pathTransformation) {
|
58
58
|
const content = await (0, utils_1.readFile)(filePath);
|
59
59
|
const { data, content: markdownContent } = (0, gray_matter_1.default)(content);
|
60
|
+
// Skip draft files
|
61
|
+
if (data.draft === true) {
|
62
|
+
return null;
|
63
|
+
}
|
60
64
|
const relativePath = path.relative(baseDir, filePath);
|
61
65
|
// Convert to URL path format (replace backslashes with forward slashes on Windows)
|
62
66
|
const normalizedPath = relativePath.replace(/\\/g, '/');
|
@@ -201,7 +205,9 @@ async function processFilesWithPatterns(context, allFiles, includePatterns = [],
|
|
201
205
|
const baseDir = isBlogFile ? path.join(siteDir, 'blog') : path.join(siteDir, docsDir);
|
202
206
|
const pathPrefix = isBlogFile ? 'blog' : 'docs';
|
203
207
|
const docInfo = await processMarkdownFile(filePath, baseDir, siteUrl, pathPrefix, context.options.pathTransformation);
|
204
|
-
|
208
|
+
if (docInfo !== null) {
|
209
|
+
processedDocs.push(docInfo);
|
210
|
+
}
|
205
211
|
}
|
206
212
|
catch (err) {
|
207
213
|
console.warn(`Error processing ${filePath}: ${err.message}`);
|
package/lib/types.d.ts
ADDED
@@ -0,0 +1,86 @@
|
|
1
|
+
/**
|
2
|
+
* Type definitions for the docusaurus-plugin-llms plugin
|
3
|
+
*/
|
4
|
+
/**
|
5
|
+
* Interface for processed document information
|
6
|
+
*/
|
7
|
+
export interface DocInfo {
|
8
|
+
title: string;
|
9
|
+
path: string;
|
10
|
+
url: string;
|
11
|
+
content: string;
|
12
|
+
description: string;
|
13
|
+
}
|
14
|
+
/**
|
15
|
+
* Interface for custom LLM file configuration
|
16
|
+
*/
|
17
|
+
export interface CustomLLMFile {
|
18
|
+
/** Name of the output file (e.g., 'llms-python.txt') */
|
19
|
+
filename: string;
|
20
|
+
/** Glob patterns for files to include */
|
21
|
+
includePatterns: string[];
|
22
|
+
/** Whether to include full content (true) or just links (false) */
|
23
|
+
fullContent: boolean;
|
24
|
+
/** Custom title for this file (defaults to site title) */
|
25
|
+
title?: string;
|
26
|
+
/** Custom description for this file (defaults to site description) */
|
27
|
+
description?: string;
|
28
|
+
/** Additional patterns to exclude (combined with global ignoreFiles) */
|
29
|
+
ignorePatterns?: string[];
|
30
|
+
/** Order patterns for controlling file ordering (similar to includeOrder) */
|
31
|
+
orderPatterns?: string[];
|
32
|
+
/** Whether to include unmatched files last (default: false) */
|
33
|
+
includeUnmatchedLast?: boolean;
|
34
|
+
/** Version information for this LLM file */
|
35
|
+
version?: string;
|
36
|
+
}
|
37
|
+
/**
|
38
|
+
* Plugin options interface
|
39
|
+
*/
|
40
|
+
export interface PluginOptions {
|
41
|
+
/** Whether to generate the llms.txt file (default: true) */
|
42
|
+
generateLLMsTxt?: boolean;
|
43
|
+
/** Whether to generate the llms-full.txt file (default: true) */
|
44
|
+
generateLLMsFullTxt?: boolean;
|
45
|
+
/** Base directory for documentation files (default: 'docs') */
|
46
|
+
docsDir?: string;
|
47
|
+
/** Array of glob patterns for files to ignore */
|
48
|
+
ignoreFiles?: string[];
|
49
|
+
/** Custom title to use in generated files (defaults to site title) */
|
50
|
+
title?: string;
|
51
|
+
/** Custom description to use in generated files (defaults to site tagline) */
|
52
|
+
description?: string;
|
53
|
+
/** Custom file name for the links file (default: 'llms.txt') */
|
54
|
+
llmsTxtFilename?: string;
|
55
|
+
/** Custom file name for the full content file (default: 'llms-full.txt') */
|
56
|
+
llmsFullTxtFilename?: string;
|
57
|
+
/** Whether to include blog content (default: false) */
|
58
|
+
includeBlog?: boolean;
|
59
|
+
/** Path transformation options for URL construction */
|
60
|
+
pathTransformation?: {
|
61
|
+
/** Path segments to ignore when constructing URLs (will be removed if found) */
|
62
|
+
ignorePaths?: string[];
|
63
|
+
/** Path segments to add when constructing URLs (will be prepended if not already present) */
|
64
|
+
addPaths?: string[];
|
65
|
+
};
|
66
|
+
/** Array of glob patterns for controlling the order of files (files will be processed in the order of patterns) */
|
67
|
+
includeOrder?: string[];
|
68
|
+
/** Whether to include files that don't match any pattern in includeOrder at the end (default: true) */
|
69
|
+
includeUnmatchedLast?: boolean;
|
70
|
+
/** Array of custom LLM file configurations */
|
71
|
+
customLLMFiles?: CustomLLMFile[];
|
72
|
+
/** Global version for all generated LLM files */
|
73
|
+
version?: string;
|
74
|
+
}
|
75
|
+
/**
|
76
|
+
* Plugin context with processed options
|
77
|
+
*/
|
78
|
+
export interface PluginContext {
|
79
|
+
siteDir: string;
|
80
|
+
outDir: string;
|
81
|
+
siteUrl: string;
|
82
|
+
docsDir: string;
|
83
|
+
docTitle: string;
|
84
|
+
docDescription: string;
|
85
|
+
options: PluginOptions;
|
86
|
+
}
|
package/lib/types.js
ADDED
package/package.json
CHANGED
@@ -1,8 +1,9 @@
|
|
1
1
|
{
|
2
2
|
"name": "docusaurus-plugin-llms",
|
3
|
-
"version": "0.1.
|
4
|
-
"description": "Docusaurus plugin for generating LLM-friendly documentation following the
|
3
|
+
"version": "0.1.5",
|
4
|
+
"description": "Docusaurus plugin for generating LLM-friendly documentation following the llmstxt.org standard",
|
5
5
|
"main": "lib/index.js",
|
6
|
+
"types": "lib/index.d.ts",
|
6
7
|
"scripts": {
|
7
8
|
"build": "tsc",
|
8
9
|
"watch": "tsc --watch",
|
package/src/generator.ts
CHANGED
@@ -60,7 +60,7 @@ ${doc.content}`;
|
|
60
60
|
|
61
61
|
> ${fileDescription}${versionInfo}
|
62
62
|
|
63
|
-
This file contains all documentation content in a single document following the
|
63
|
+
This file contains all documentation content in a single document following the llmstxt.org standard.
|
64
64
|
|
65
65
|
${fullContentSections.join('\n\n---\n\n')}
|
66
66
|
`;
|
@@ -79,7 +79,7 @@ ${fullContentSections.join('\n\n---\n\n')}
|
|
79
79
|
|
80
80
|
> ${fileDescription}${versionInfo}
|
81
81
|
|
82
|
-
This file contains links to documentation sections following the
|
82
|
+
This file contains links to documentation sections following the llmstxt.org standard.
|
83
83
|
|
84
84
|
## Table of Contents
|
85
85
|
|
package/src/index.ts
CHANGED
@@ -1,5 +1,5 @@
|
|
1
1
|
/**
|
2
|
-
* @fileoverview Docusaurus plugin that generates LLM-friendly documentation following the
|
2
|
+
* @fileoverview Docusaurus plugin that generates LLM-friendly documentation following the llmstxt.org standard.
|
3
3
|
*
|
4
4
|
* This plugin creates two files:
|
5
5
|
* - llms.txt: Contains links to all sections of documentation
|
@@ -15,7 +15,7 @@ import { collectDocFiles, generateStandardLLMFiles, generateCustomLLMFiles } fro
|
|
15
15
|
|
16
16
|
/**
|
17
17
|
* A Docusaurus plugin to generate LLM-friendly documentation following
|
18
|
-
* the
|
18
|
+
* the llmstxt.org standard
|
19
19
|
*
|
20
20
|
* @param context - Docusaurus context
|
21
21
|
* @param options - Plugin options
|
@@ -112,4 +112,6 @@ export default function docusaurusPluginLLMs(
|
|
112
112
|
}
|
113
113
|
},
|
114
114
|
};
|
115
|
-
}
|
115
|
+
}
|
116
|
+
|
117
|
+
export type { PluginOptions };
|
package/src/processor.ts
CHANGED
@@ -31,10 +31,15 @@ export async function processMarkdownFile(
|
|
31
31
|
ignorePaths?: string[];
|
32
32
|
addPaths?: string[];
|
33
33
|
}
|
34
|
-
): Promise<DocInfo> {
|
34
|
+
): Promise<DocInfo | null> {
|
35
35
|
const content = await readFile(filePath);
|
36
36
|
const { data, content: markdownContent } = matter(content);
|
37
37
|
|
38
|
+
// Skip draft files
|
39
|
+
if (data.draft === true) {
|
40
|
+
return null;
|
41
|
+
}
|
42
|
+
|
38
43
|
const relativePath = path.relative(baseDir, filePath);
|
39
44
|
// Convert to URL path format (replace backslashes with forward slashes on Windows)
|
40
45
|
const normalizedPath = relativePath.replace(/\\/g, '/');
|
@@ -226,7 +231,9 @@ export async function processFilesWithPatterns(
|
|
226
231
|
pathPrefix,
|
227
232
|
context.options.pathTransformation
|
228
233
|
);
|
229
|
-
|
234
|
+
if (docInfo !== null) {
|
235
|
+
processedDocs.push(docInfo);
|
236
|
+
}
|
230
237
|
} catch (err: any) {
|
231
238
|
console.warn(`Error processing ${filePath}: ${err.message}`);
|
232
239
|
}
|