docusaurus-plugin-llms 0.2.0 → 0.2.2

package/README.md

@@ -137,6 +137,7 @@ module.exports = {
 | `version`                        | string   | `undefined`       | Global version to include in all generated files              |
 | `customLLMFiles`                 | array    | `[]`              | Array of custom LLM file configurations                       |
 | `generateMarkdownFiles`          | boolean  | `false`           | Generate individual markdown files and link to them from llms.txt |
+| `keepFrontMatter`                | string[] | []                | Preserve selected front matter items when generating individual markdown files
 | `rootContent`                    | string   | (see below)       | Custom content to include at the root level of llms.txt       |
 | `fullRootContent`                | string   | (see below)       | Custom content to include at the root level of llms-full.txt  |
 

package/lib/generator.d.ts

@@ -18,9 +18,11 @@ export declare function generateLLMFile(docs: DocInfo[], outputPath: string, fil
  * @param docs - Processed document information
  * @param outputDir - Directory to write the markdown files
  * @param siteUrl - Base site URL
+ * @param docsDir - The configured docs directory name (e.g., 'docs', 'documentation', etc.)
+ * @param keepFrontMatter - Array of frontmatter keys to preserve in generated files
  * @returns Updated docs with new URLs pointing to generated markdown files
  */
-export declare function generateIndividualMarkdownFiles(docs: DocInfo[], outputDir: string, siteUrl: string): Promise<DocInfo[]>;
+export declare function generateIndividualMarkdownFiles(docs: DocInfo[], outputDir: string, siteUrl: string, docsDir?: string, keepFrontMatter?: string[]): Promise<DocInfo[]>;
 /**
  * Generate standard LLM files (llms.txt and llms-full.txt)
  * @param context - Plugin context

package/lib/generator.js

@@ -85,29 +85,18 @@ async function generateLLMFile(docs, outputPath, fileTitle, fileDescription, inc
             // Check if the first line is a heading that matches our title
             const headingMatch = firstLine.match(/^#+\s+(.+)$/);
             const firstHeadingText = headingMatch ? headingMatch[1].trim() : null;
-            // Determine the header text to use (original title or make it unique)
-            let headerText = doc.title;
-            let uniqueHeader = headerText;
-            let counter = 1;
-            // If this header has been used before, make it unique by adding a suffix
-            while (usedHeaders.has(uniqueHeader.toLowerCase())) {
-                counter++;
+            // Generate unique header using the utility function
+            const uniqueHeader = (0, utils_1.ensureUniqueIdentifier)(doc.title, usedHeaders, (counter, base) => {
                 // Try to make it more descriptive by adding the file path info if available
                 if (doc.path && counter === 2) {
                     const pathParts = doc.path.split('/');
                     const folderName = pathParts.length > 1 ? pathParts[pathParts.length - 2] : '';
                     if (folderName) {
-                        uniqueHeader = `${headerText} (${folderName.charAt(0).toUpperCase() + folderName.slice(1)})`;
+                        return `(${folderName.charAt(0).toUpperCase() + folderName.slice(1)})`;
                     }
-                    else {
-                        uniqueHeader = `${headerText} (${counter})`;
-                    }
-                }
-                else {
-                    uniqueHeader = `${headerText} (${counter})`;
                 }
-            }
-            usedHeaders.add(uniqueHeader.toLowerCase());
+                return `(${counter})`;
+            });
             if (firstHeadingText === doc.title) {
                 // Content already has the same heading, replace it with our unique header if needed
                 if (uniqueHeader !== doc.title) {
@@ -133,14 +122,8 @@ ${doc.content}`;
         });
         // Use custom root content or default message
         const rootContent = customRootContent || 'This file contains all documentation content in a single document following the llmstxt.org standard.';
-        const llmFileContent = `# ${fileTitle}
-
-> ${fileDescription}${versionInfo}
-
-${rootContent}
-
-${fullContentSections.join('\n\n---\n\n')}
-`;
+        const llmFileContent = (0, utils_1.createMarkdownContent)(fileTitle, `${fileDescription}${versionInfo}`, `${rootContent}\n\n${fullContentSections.join('\n\n---\n\n')}`, true // include metadata (description)
+        );
         await (0, utils_1.writeFile)(outputPath, llmFileContent);
     }
     else {
@@ -152,16 +135,8 @@ ${fullContentSections.join('\n\n---\n\n')}
         });
         // Use custom root content or default message
         const rootContent = customRootContent || 'This file contains links to documentation sections following the llmstxt.org standard.';
-        const llmFileContent = `# ${fileTitle}
-
-> ${fileDescription}${versionInfo}
-
-${rootContent}
-
-## Table of Contents
-
-${tocItems.join('\n')}
-`;
+        const llmFileContent = (0, utils_1.createMarkdownContent)(fileTitle, `${fileDescription}${versionInfo}`, `${rootContent}\n\n## Table of Contents\n\n${tocItems.join('\n')}`, true // include metadata (description)
+        );
         await (0, utils_1.writeFile)(outputPath, llmFileContent);
     }
     console.log(`Generated: ${outputPath}`);
@@ -171,52 +146,65 @@ ${tocItems.join('\n')}
  * @param docs - Processed document information
  * @param outputDir - Directory to write the markdown files
  * @param siteUrl - Base site URL
+ * @param docsDir - The configured docs directory name (e.g., 'docs', 'documentation', etc.)
+ * @param keepFrontMatter - Array of frontmatter keys to preserve in generated files
  * @returns Updated docs with new URLs pointing to generated markdown files
  */
-async function generateIndividualMarkdownFiles(docs, outputDir, siteUrl) {
+async function generateIndividualMarkdownFiles(docs, outputDir, siteUrl, docsDir = 'docs', keepFrontMatter = []) {
     const updatedDocs = [];
-    // Create a map to ensure unique filenames
-    const usedFilenames = new Set();
+    const usedPaths = new Set();
     for (const doc of docs) {
-        // Generate a filename from the document title or URL path
-        let baseFilename = doc.title
-            .toLowerCase()
-            .replace(/[^a-z0-9]+/g, '-')
-            .replace(/^-+|-+$/g, '');
-        // Fallback to URL path if title generates empty filename
-        if (!baseFilename) {
-            baseFilename = doc.path
-                .replace(/^\/+|\/+$/g, '') // Remove leading/trailing slashes
-                .replace(/\//g, '-')
-                .replace(/[^a-z0-9-]/gi, '-')
-                .toLowerCase();
+        // Use the original path structure, cleaning it up for file system use
+        let relativePath = doc.path
+            .replace(/^\/+/, '') // Remove leading slashes
+            .replace(/\.mdx?$/, '.md'); // Ensure .md extension
+        relativePath = relativePath
+            .replace(new RegExp(`^${docsDir.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}/`), ''); // Remove configured docs dir prefix
+        // If path is empty or invalid, create a fallback path
+        if (!relativePath || relativePath === '.md') {
+            const sanitizedTitle = (0, utils_1.sanitizeForFilename)(doc.title, 'untitled');
+            relativePath = `${sanitizedTitle}.md`;
         }
-        // Ensure filename uniqueness
-        let filename = `${baseFilename}.md`;
+        // Ensure path uniqueness
+        let uniquePath = relativePath;
         let counter = 1;
-        while (usedFilenames.has(filename)) {
-            filename = `${baseFilename}-${counter}.md`;
+        while (usedPaths.has(uniquePath.toLowerCase())) {
             counter++;
+            const pathParts = relativePath.split('.');
+            const extension = pathParts.pop() || 'md';
+            const basePath = pathParts.join('.');
+            uniquePath = `${basePath}-${counter}.${extension}`;
         }
-        usedFilenames.add(filename);
-        // Create markdown content following llmstxt.org standard
-        const markdownContent = `# ${doc.title}
-
-> ${doc.description}
-
-${doc.content}
-`;
+        usedPaths.add(uniquePath.toLowerCase());
+        // Create the full file path and ensure directory exists
+        const fullPath = path.join(outputDir, uniquePath);
+        const directory = path.dirname(fullPath);
+        // Create directory structure if it doesn't exist
+        await fs.mkdir(directory, { recursive: true });
+        // Extract preserved frontmatter if specified
+        let preservedFrontMatter = {};
+        if (keepFrontMatter.length > 0 && doc.frontMatter) {
+            for (const key of keepFrontMatter) {
+                if (key in doc.frontMatter) {
+                    preservedFrontMatter[key] = doc.frontMatter[key];
+                }
+            }
+        }
+        // Create markdown content using the utility function
+        const markdownContent = (0, utils_1.createMarkdownContent)(doc.title, doc.description, doc.content, true, // includeMetadata
+        Object.keys(preservedFrontMatter).length > 0 ? preservedFrontMatter : undefined);
         // Write the markdown file
-        const markdownPath = path.join(outputDir, filename);
-        await (0, utils_1.writeFile)(markdownPath, markdownContent);
+        await (0, utils_1.writeFile)(fullPath, markdownContent);
         // Create updated DocInfo with new URL pointing to the generated markdown file
-        const newUrl = `${siteUrl}/${filename}`;
+        // Convert file path to URL path (use forward slashes)
+        const urlPath = uniquePath.replace(/\\/g, '/');
+        const newUrl = `${siteUrl}/${urlPath}`;
         updatedDocs.push({
             ...doc,
             url: newUrl,
-            path: `/${filename}` // Update path to the new markdown file
+            path: `/${urlPath}` // Update path to the new markdown file
         });
-        console.log(`Generated markdown file: ${filename}`);
+        console.log(`Generated markdown file: ${uniquePath}`);
     }
     return updatedDocs;
 }
@@ -239,7 +227,7 @@ async function generateStandardLLMFiles(context, allDocFiles) {
     // Generate individual markdown files if requested
     if (generateMarkdownFiles && processedDocs.length > 0) {
         console.log('Generating individual markdown files...');
-        processedDocs = await generateIndividualMarkdownFiles(processedDocs, outDir, siteUrl);
+        processedDocs = await generateIndividualMarkdownFiles(processedDocs, outDir, siteUrl, context.docsDir, context.options.keepFrontMatter || []);
     }
     // Generate llms.txt
     if (generateLLMsTxt) {
@@ -279,7 +267,7 @@ async function generateCustomLLMFiles(context, allDocFiles) {
             // Generate individual markdown files if requested
             if (generateMarkdownFiles) {
                 console.log(`Generating individual markdown files for custom file: ${customFile.filename}...`);
-                customDocs = await generateIndividualMarkdownFiles(customDocs, outDir, siteUrl);
+                customDocs = await generateIndividualMarkdownFiles(customDocs, outDir, siteUrl, context.docsDir, context.options.keepFrontMatter || []);
             }
             // Use custom title/description or fall back to defaults
             const customTitle = customFile.title || docTitle;

package/lib/index.js

@@ -21,7 +21,7 @@ const generator_1 = require("./generator");
  */
 function docusaurusPluginLLMs(context, options = {}) {
     // Set default options
-    const { generateLLMsTxt = true, generateLLMsFullTxt = true, docsDir = 'docs', ignoreFiles = [], title, description, llmsTxtFilename = 'llms.txt', llmsFullTxtFilename = 'llms-full.txt', includeBlog = false, pathTransformation, includeOrder = [], includeUnmatchedLast = true, customLLMFiles = [], excludeImports = false, removeDuplicateHeadings = false, generateMarkdownFiles = false, rootContent, fullRootContent, } = options;
+    const { generateLLMsTxt = true, generateLLMsFullTxt = true, docsDir = 'docs', ignoreFiles = [], title, description, llmsTxtFilename = 'llms.txt', llmsFullTxtFilename = 'llms-full.txt', includeBlog = false, pathTransformation, includeOrder = [], includeUnmatchedLast = true, customLLMFiles = [], excludeImports = false, removeDuplicateHeadings = false, generateMarkdownFiles = false, keepFrontMatter = [], rootContent, fullRootContent, } = options;
     const { siteDir, siteConfig, outDir, } = context;
     // Build the site URL with proper trailing slash
     const siteUrl = siteConfig.url + (siteConfig.baseUrl.endsWith('/')
@@ -52,6 +52,7 @@ function docusaurusPluginLLMs(context, options = {}) {
             excludeImports,
             removeDuplicateHeadings,
             generateMarkdownFiles,
+            keepFrontMatter,
             rootContent,
             fullRootContent,
         }

package/lib/processor.js

@@ -76,10 +76,18 @@ async function processMarkdownFile(filePath, baseDir, siteUrl, pathPrefix = 'doc
         // Convert .md extension to appropriate path
         const linkPathBase = normalizedPath.replace(/\.mdx?$/, '');
         // Handle index files specially
-        const linkPath = linkPathBase.endsWith('index')
+        let linkPath = linkPathBase.endsWith('index')
             ? linkPathBase.replace(/\/index$/, '')
             : linkPathBase;
-        // Apply path transformations to the link path
+        // linkPath might include the pathPrefix (e.g., "docs/api/core")
+        // We need to remove the pathPrefix before applying transformations, then add it back later
+        if (pathPrefix && linkPath.startsWith(`${pathPrefix}/`)) {
+            linkPath = linkPath.substring(`${pathPrefix}/`.length);
+        }
+        else if (pathPrefix && linkPath === pathPrefix) {
+            linkPath = '';
+        }
+        // Apply path transformations to the clean link path (without pathPrefix)
         const transformedLinkPath = (0, utils_1.applyPathTransformations)(linkPath, pathTransformation);
         // Also apply path transformations to the pathPrefix if it's not empty
         // This allows removing 'docs' from the path when specified in ignorePaths
@@ -153,6 +161,7 @@ async function processMarkdownFile(filePath, baseDir, siteUrl, pathPrefix = 'doc
         url: fullUrl,
         content: cleanedContent,
         description: description || '',
+        frontMatter: data,
     };
 }
 /**
@@ -212,7 +221,8 @@ async function processFilesWithPatterns(context, allFiles, includePatterns = [],
         try {
             // Determine if this is a blog or docs file
             const isBlogFile = filePath.includes(path.join(siteDir, 'blog'));
-            const baseDir = isBlogFile ? path.join(siteDir, 'blog') : path.join(siteDir, docsDir);
+            // Use siteDir as baseDir to preserve full directory structure (docs/path/file.md instead of just path/file.md)
+            const baseDir = siteDir;
             const pathPrefix = isBlogFile ? 'blog' : 'docs';
             // Try to find the resolved URL for this file from the route map
             let resolvedUrl;

package/lib/types.d.ts

@@ -11,6 +11,7 @@ export interface DocInfo {
     url: string;
     content: string;
     description: string;
+    frontMatter?: Record<string, any>;
 }
 /**
  * Interface for custom LLM file configuration
@@ -80,6 +81,8 @@ export interface PluginOptions {
     removeDuplicateHeadings?: boolean;
     /** Whether to generate individual markdown files and link to them from llms.txt instead of original docs (default: false) */
     generateMarkdownFiles?: boolean;
+    /** Array of frontmatter keys to preserve in generated individual markdown files (only used when generateMarkdownFiles is true) */
+    keepFrontMatter?: string[];
     /** Custom content to include at the root level of llms.txt (after title/description, before TOC) */
     rootContent?: string;
     /** Custom content to include at the root level of llms-full.txt (after title/description, before content sections) */

package/lib/utils.d.ts

@@ -60,3 +60,28 @@ export declare function cleanMarkdownContent(content: string, excludeImports?: b
  * @returns Transformed URL path
  */
 export declare function applyPathTransformations(urlPath: string, pathTransformation?: PluginOptions['pathTransformation']): string;
+/**
+ * Sanitize a string to create a safe filename
+ * @param input - Input string (typically a title)
+ * @param fallback - Fallback string if input becomes empty after sanitization
+ * @returns Sanitized filename (without extension)
+ */
+export declare function sanitizeForFilename(input: string, fallback?: string): string;
+/**
+ * Ensure a unique identifier from a set of used identifiers
+ * @param baseIdentifier - Base identifier to make unique
+ * @param usedIdentifiers - Set of already used identifiers
+ * @param suffix - Suffix pattern (default: number in parentheses)
+ * @returns Unique identifier
+ */
+export declare function ensureUniqueIdentifier(baseIdentifier: string, usedIdentifiers: Set<string>, suffix?: (counter: number, base: string) => string): string;
+/**
+ * Create standardized markdown content template
+ * @param title - Document title
+ * @param description - Document description
+ * @param content - Document content
+ * @param includeMetadata - Whether to include description metadata
+ * @param frontMatter - Optional frontmatter to include at the top
+ * @returns Formatted markdown content
+ */
+export declare function createMarkdownContent(title: string, description?: string, content?: string, includeMetadata?: boolean, frontMatter?: Record<string, any>): string;

package/lib/utils.js

@@ -47,10 +47,14 @@ exports.extractTitle = extractTitle;
 exports.resolvePartialImports = resolvePartialImports;
 exports.cleanMarkdownContent = cleanMarkdownContent;
 exports.applyPathTransformations = applyPathTransformations;
+exports.sanitizeForFilename = sanitizeForFilename;
+exports.ensureUniqueIdentifier = ensureUniqueIdentifier;
+exports.createMarkdownContent = createMarkdownContent;
 const fs = __importStar(require("fs/promises"));
 const path = __importStar(require("path"));
 const minimatch_1 = require("minimatch");
 const gray_matter_1 = __importDefault(require("gray-matter"));
+const YAML = __importStar(require("yaml"));
 /**
  * Write content to a file
  * @param filePath - Path to write the file to
@@ -129,7 +133,7 @@ function extractTitle(data, content, filePath) {
     // Finally use filename
     return path.basename(filePath, path.extname(filePath))
         .replace(/-/g, ' ')
-        .replace(/\b\w/g, c => c.toUpperCase());
+        .replace(/\b\w/g, (c) => c.toUpperCase());
 }
 /**
  * Resolve and inline partial imports in markdown content
@@ -286,3 +290,57 @@ function applyPathTransformations(urlPath, pathTransformation) {
     }
     return transformedPath;
 }
+/**
+ * Sanitize a string to create a safe filename
+ * @param input - Input string (typically a title)
+ * @param fallback - Fallback string if input becomes empty after sanitization
+ * @returns Sanitized filename (without extension)
+ */
+function sanitizeForFilename(input, fallback = 'untitled') {
+    if (!input)
+        return fallback;
+    const sanitized = input
+        .toLowerCase()
+        .replace(/[^a-z0-9]+/g, '-')
+        .replace(/^-+|-+$/g, '');
+    return sanitized || fallback;
+}
+/**
+ * Ensure a unique identifier from a set of used identifiers
+ * @param baseIdentifier - Base identifier to make unique
+ * @param usedIdentifiers - Set of already used identifiers
+ * @param suffix - Suffix pattern (default: number in parentheses)
+ * @returns Unique identifier
+ */
+function ensureUniqueIdentifier(baseIdentifier, usedIdentifiers, suffix = (counter) => `(${counter})`) {
+    let uniqueIdentifier = baseIdentifier;
+    let counter = 1;
+    while (usedIdentifiers.has(uniqueIdentifier.toLowerCase())) {
+        counter++;
+        uniqueIdentifier = `${baseIdentifier}${suffix(counter, baseIdentifier)}`;
+    }
+    usedIdentifiers.add(uniqueIdentifier.toLowerCase());
+    return uniqueIdentifier;
+}
+/**
+ * Create standardized markdown content template
+ * @param title - Document title
+ * @param description - Document description
+ * @param content - Document content
+ * @param includeMetadata - Whether to include description metadata
+ * @param frontMatter - Optional frontmatter to include at the top
+ * @returns Formatted markdown content
+ */
+function createMarkdownContent(title, description = '', content = '', includeMetadata = true, frontMatter) {
+    let result = '';
+    // Add frontmatter if provided
+    if (frontMatter && Object.keys(frontMatter).length > 0) {
+        result += '---\n';
+        result += YAML.stringify(frontMatter);
+        result += '---\n\n';
+    }
+    const descriptionLine = includeMetadata && description ? `\n\n> ${description}\n` : '\n';
+    result += `# ${title}${descriptionLine}
+${content}`.trim() + '\n';
+    return result;
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "docusaurus-plugin-llms",
-  "version": "0.2.0",
+  "version": "0.2.2",
   "description": "Docusaurus plugin for generating LLM-friendly documentation following the llmstxt.org standard",
   "main": "lib/index.js",
   "types": "lib/index.d.ts",
@@ -38,13 +38,15 @@
   "license": "MIT",
   "dependencies": {
     "gray-matter": "^4.0.3",
-    "minimatch": "^9.0.3"
+    "minimatch": "^9.0.3",
+    "yaml": "^2.8.1"
   },
   "peerDependencies": {
     "@docusaurus/core": "^3.0.0"
   },
   "devDependencies": {
     "@docusaurus/types": "^3.0.0",
+    "@types/js-yaml": "^4.0.9",
     "@types/minimatch": "^5.1.2",
     "@types/node": "^20.6.0",
     "typescript": "^5.2.2"

package/src/generator.ts

@@ -5,7 +5,13 @@
 import * as path from 'path';
 import * as fs from 'fs/promises';
 import { DocInfo, PluginContext, CustomLLMFile } from './types';
-import { writeFile, readMarkdownFiles } from './utils';
+import { 
+  writeFile, 
+  readMarkdownFiles, 
+  sanitizeForFilename, 
+  ensureUniqueIdentifier, 
+  createMarkdownContent 
+} from './utils';
 import { processFilesWithPatterns } from './processor';
 
 /**
@@ -62,29 +68,22 @@ export async function generateLLMFile(
       const headingMatch = firstLine.match(/^#+\s+(.+)$/);
       const firstHeadingText = headingMatch ? headingMatch[1].trim() : null;
       
-      // Determine the header text to use (original title or make it unique)
-      let headerText = doc.title;
-      let uniqueHeader = headerText;
-      let counter = 1;
-      
-      // If this header has been used before, make it unique by adding a suffix
-      while (usedHeaders.has(uniqueHeader.toLowerCase())) {
-        counter++;
-        // Try to make it more descriptive by adding the file path info if available
-        if (doc.path && counter === 2) {
-          const pathParts = doc.path.split('/');
-          const folderName = pathParts.length > 1 ? pathParts[pathParts.length - 2] : '';
-          if (folderName) {
-            uniqueHeader = `${headerText} (${folderName.charAt(0).toUpperCase() + folderName.slice(1)})`;
-          } else {
-            uniqueHeader = `${headerText} (${counter})`;
+      // Generate unique header using the utility function
+      const uniqueHeader = ensureUniqueIdentifier(
+        doc.title, 
+        usedHeaders, 
+        (counter, base) => {
+          // Try to make it more descriptive by adding the file path info if available
+          if (doc.path && counter === 2) {
+            const pathParts = doc.path.split('/');
+            const folderName = pathParts.length > 1 ? pathParts[pathParts.length - 2] : '';
+            if (folderName) {
+              return `(${folderName.charAt(0).toUpperCase() + folderName.slice(1)})`;
+            }
           }
-        } else {
-          uniqueHeader = `${headerText} (${counter})`;
+          return `(${counter})`;
         }
-      }
-      
-      usedHeaders.add(uniqueHeader.toLowerCase());
+      );
       
       if (firstHeadingText === doc.title) {
         // Content already has the same heading, replace it with our unique header if needed
@@ -111,14 +110,12 @@ ${doc.content}`;
     // Use custom root content or default message
     const rootContent = customRootContent || 'This file contains all documentation content in a single document following the llmstxt.org standard.';
     
-    const llmFileContent = `# ${fileTitle}
-
-> ${fileDescription}${versionInfo}
-
-${rootContent}
-
-${fullContentSections.join('\n\n---\n\n')}
-`;
+    const llmFileContent = createMarkdownContent(
+      fileTitle,
+      `${fileDescription}${versionInfo}`,
+      `${rootContent}\n\n${fullContentSections.join('\n\n---\n\n')}`,
+      true // include metadata (description)
+    );
 
     await writeFile(outputPath, llmFileContent);
   } else {
@@ -133,16 +130,12 @@ ${fullContentSections.join('\n\n---\n\n')}
     // Use custom root content or default message
     const rootContent = customRootContent || 'This file contains links to documentation sections following the llmstxt.org standard.';
     
-    const llmFileContent = `# ${fileTitle}
-
-> ${fileDescription}${versionInfo}
-
-${rootContent}
-
-## Table of Contents
-
-${tocItems.join('\n')}
-`;
+    const llmFileContent = createMarkdownContent(
+      fileTitle,
+      `${fileDescription}${versionInfo}`,
+      `${rootContent}\n\n## Table of Contents\n\n${tocItems.join('\n')}`,
+      true // include metadata (description)
+    );
 
     await writeFile(outputPath, llmFileContent);
   }
@@ -155,65 +148,90 @@ ${tocItems.join('\n')}
  * @param docs - Processed document information  
  * @param outputDir - Directory to write the markdown files
  * @param siteUrl - Base site URL
+ * @param docsDir - The configured docs directory name (e.g., 'docs', 'documentation', etc.)
+ * @param keepFrontMatter - Array of frontmatter keys to preserve in generated files
  * @returns Updated docs with new URLs pointing to generated markdown files
  */
 export async function generateIndividualMarkdownFiles(
   docs: DocInfo[],
   outputDir: string,
-  siteUrl: string
+  siteUrl: string,
+  docsDir: string = 'docs',
+  keepFrontMatter: string[] = []
 ): Promise<DocInfo[]> {
   const updatedDocs: DocInfo[] = [];
+  const usedPaths = new Set<string>();
   
-  // Create a map to ensure unique filenames
-  const usedFilenames = new Set<string>();
   
   for (const doc of docs) {
-    // Generate a filename from the document title or URL path
-    let baseFilename = doc.title
-      .toLowerCase()
-      .replace(/[^a-z0-9]+/g, '-')
-      .replace(/^-+|-+$/g, '');
+    // Use the original path structure, cleaning it up for file system use
+    let relativePath = doc.path
+      .replace(/^\/+/, '') // Remove leading slashes
+      .replace(/\.mdx?$/, '.md'); // Ensure .md extension
+    
     
-    // Fallback to URL path if title generates empty filename
-    if (!baseFilename) {
-      baseFilename = doc.path
-        .replace(/^\/+|\/+$/g, '') // Remove leading/trailing slashes
-        .replace(/\//g, '-')
-        .replace(/[^a-z0-9-]/gi, '-')
-        .toLowerCase();
+    relativePath = relativePath
+      .replace(new RegExp(`^${docsDir.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}/`), '');// Remove configured docs dir prefix
+    
+    // If path is empty or invalid, create a fallback path
+    if (!relativePath || relativePath === '.md') {
+      const sanitizedTitle = sanitizeForFilename(doc.title, 'untitled');
+      relativePath = `${sanitizedTitle}.md`;
     }
     
-    // Ensure filename uniqueness
-    let filename = `${baseFilename}.md`;
+    // Ensure path uniqueness
+    let uniquePath = relativePath;
     let counter = 1;
-    while (usedFilenames.has(filename)) {
-      filename = `${baseFilename}-${counter}.md`;
+    while (usedPaths.has(uniquePath.toLowerCase())) {
       counter++;
+      const pathParts = relativePath.split('.');
+      const extension = pathParts.pop() || 'md';
+      const basePath = pathParts.join('.');
+      uniquePath = `${basePath}-${counter}.${extension}`;
     }
-    usedFilenames.add(filename);
+    usedPaths.add(uniquePath.toLowerCase());
     
-    // Create markdown content following llmstxt.org standard
-    const markdownContent = `# ${doc.title}
-
-> ${doc.description}
+    // Create the full file path and ensure directory exists
+    const fullPath = path.join(outputDir, uniquePath);
+    const directory = path.dirname(fullPath);
+    
+    // Create directory structure if it doesn't exist
+    await fs.mkdir(directory, { recursive: true });
+    
+    // Extract preserved frontmatter if specified
+    let preservedFrontMatter: Record<string, any> = {};
+    if (keepFrontMatter.length > 0 && doc.frontMatter) {
+      for (const key of keepFrontMatter) {
+        if (key in doc.frontMatter) {
+          preservedFrontMatter[key] = doc.frontMatter[key];
+        }
+      }
+    }
 
-${doc.content}
-`;
+    // Create markdown content using the utility function
+    const markdownContent = createMarkdownContent(
+      doc.title, 
+      doc.description, 
+      doc.content, 
+      true, // includeMetadata
+      Object.keys(preservedFrontMatter).length > 0 ? preservedFrontMatter : undefined
+    );
     
     // Write the markdown file
-    const markdownPath = path.join(outputDir, filename);
-    await writeFile(markdownPath, markdownContent);
+    await writeFile(fullPath, markdownContent);
     
     // Create updated DocInfo with new URL pointing to the generated markdown file
-    const newUrl = `${siteUrl}/${filename}`;
+    // Convert file path to URL path (use forward slashes)
+    const urlPath = uniquePath.replace(/\\/g, '/');
+    const newUrl = `${siteUrl}/${urlPath}`;
     
     updatedDocs.push({
       ...doc,
       url: newUrl,
-      path: `/${filename}` // Update path to the new markdown file
+      path: `/${urlPath}` // Update path to the new markdown file
     });
     
-    console.log(`Generated markdown file: ${filename}`);
+    console.log(`Generated markdown file: ${uniquePath}`);
   }
   
   return updatedDocs;
@@ -271,7 +289,9 @@ export async function generateStandardLLMFiles(
     processedDocs = await generateIndividualMarkdownFiles(
       processedDocs,
       outDir,
-      siteUrl
+      siteUrl,
+      context.docsDir,
+      context.options.keepFrontMatter || []
     );
   }
   
@@ -348,7 +368,9 @@ export async function generateCustomLLMFiles(
         customDocs = await generateIndividualMarkdownFiles(
           customDocs,
           outDir,
-          siteUrl
+          siteUrl,
+          context.docsDir,
+          context.options.keepFrontMatter || []
         );
       }
       

package/src/index.ts

@@ -43,6 +43,7 @@ export default function docusaurusPluginLLMs(
     excludeImports = false,
     removeDuplicateHeadings = false,
     generateMarkdownFiles = false,
+    keepFrontMatter = [],
     rootContent,
     fullRootContent,
   } = options;
@@ -85,6 +86,7 @@ export default function docusaurusPluginLLMs(
       excludeImports,
       removeDuplicateHeadings,
       generateMarkdownFiles,
+      keepFrontMatter,
       rootContent,
       fullRootContent,
     }

package/src/processor.ts

@@ -62,11 +62,19 @@ export async function processMarkdownFile(
     const linkPathBase = normalizedPath.replace(/\.mdx?$/, '');
     
     // Handle index files specially
-    const linkPath = linkPathBase.endsWith('index') 
+    let linkPath = linkPathBase.endsWith('index') 
       ? linkPathBase.replace(/\/index$/, '') 
       : linkPathBase;
     
-    // Apply path transformations to the link path
+    // linkPath might include the pathPrefix (e.g., "docs/api/core")
+    // We need to remove the pathPrefix before applying transformations, then add it back later
+    if (pathPrefix && linkPath.startsWith(`${pathPrefix}/`)) {
+      linkPath = linkPath.substring(`${pathPrefix}/`.length);
+    } else if (pathPrefix && linkPath === pathPrefix) {
+      linkPath = '';
+    }
+    
+    // Apply path transformations to the clean link path (without pathPrefix)
     const transformedLinkPath = applyPathTransformations(linkPath, pathTransformation);
     
     // Also apply path transformations to the pathPrefix if it's not empty
@@ -156,6 +164,7 @@ export async function processMarkdownFile(
     url: fullUrl,
     content: cleanedContent,
     description: description || '',
+    frontMatter: data,
   };
 }
 
@@ -236,7 +245,8 @@ export async function processFilesWithPatterns(
     try {
       // Determine if this is a blog or docs file
       const isBlogFile = filePath.includes(path.join(siteDir, 'blog'));
-      const baseDir = isBlogFile ? path.join(siteDir, 'blog') : path.join(siteDir, docsDir);
+      // Use siteDir as baseDir to preserve full directory structure (docs/path/file.md instead of just path/file.md)
+      const baseDir = siteDir;
       const pathPrefix = isBlogFile ? 'blog' : 'docs';
       
       // Try to find the resolved URL for this file from the route map

package/src/types.ts

@@ -13,6 +13,7 @@ export interface DocInfo {
   url: string;
   content: string;
   description: string;
+  frontMatter?: Record<string, any>;
 }
 
 /**
@@ -109,7 +110,10 @@ export interface PluginOptions {
   
   /** Whether to generate individual markdown files and link to them from llms.txt instead of original docs (default: false) */
   generateMarkdownFiles?: boolean;
-  
+
+  /** Array of frontmatter keys to preserve in generated individual markdown files (only used when generateMarkdownFiles is true) */
+  keepFrontMatter?: string[];
+
   /** Custom content to include at the root level of llms.txt (after title/description, before TOC) */
   rootContent?: string;
   

package/src/utils.ts

@@ -6,6 +6,7 @@ import * as fs from 'fs/promises';
 import * as path from 'path';
 import { minimatch } from 'minimatch';
 import matter from 'gray-matter';
+import * as YAML from 'yaml';
 import { PluginOptions } from './types';
 
 /**
@@ -99,7 +100,7 @@ export function extractTitle(data: any, content: string, filePath: string): stri
   // Finally use filename
   return path.basename(filePath, path.extname(filePath))
     .replace(/-/g, ' ')
-    .replace(/\b\w/g, c => c.toUpperCase());
+    .replace(/\b\w/g, (c: string) => c.toUpperCase());
 }
 
 /**
@@ -290,4 +291,78 @@ export function applyPathTransformations(
   }
   
   return transformedPath;
+}
+
+/**
+ * Sanitize a string to create a safe filename
+ * @param input - Input string (typically a title)
+ * @param fallback - Fallback string if input becomes empty after sanitization
+ * @returns Sanitized filename (without extension)
+ */
+export function sanitizeForFilename(input: string, fallback: string = 'untitled'): string {
+  if (!input) return fallback;
+  
+  const sanitized = input
+    .toLowerCase()
+    .replace(/[^a-z0-9]+/g, '-')
+    .replace(/^-+|-+$/g, '');
+  
+  return sanitized || fallback;
+}
+
+/**
+ * Ensure a unique identifier from a set of used identifiers
+ * @param baseIdentifier - Base identifier to make unique
+ * @param usedIdentifiers - Set of already used identifiers
+ * @param suffix - Suffix pattern (default: number in parentheses)
+ * @returns Unique identifier
+ */
+export function ensureUniqueIdentifier(
+  baseIdentifier: string, 
+  usedIdentifiers: Set<string>,
+  suffix: (counter: number, base: string) => string = (counter) => `(${counter})`
+): string {
+  let uniqueIdentifier = baseIdentifier;
+  let counter = 1;
+  
+  while (usedIdentifiers.has(uniqueIdentifier.toLowerCase())) {
+    counter++;
+    uniqueIdentifier = `${baseIdentifier}${suffix(counter, baseIdentifier)}`;
+  }
+  
+  usedIdentifiers.add(uniqueIdentifier.toLowerCase());
+  return uniqueIdentifier;
+}
+
+/**
+ * Create standardized markdown content template
+ * @param title - Document title
+ * @param description - Document description
+ * @param content - Document content
+ * @param includeMetadata - Whether to include description metadata
+ * @param frontMatter - Optional frontmatter to include at the top
+ * @returns Formatted markdown content
+ */
+export function createMarkdownContent(
+  title: string, 
+  description: string = '', 
+  content: string = '',
+  includeMetadata: boolean = true,
+  frontMatter?: Record<string, any>
+): string {
+  let result = '';
+  
+  // Add frontmatter if provided
+  if (frontMatter && Object.keys(frontMatter).length > 0) {
+    result += '---\n';
+    result += YAML.stringify(frontMatter);
+    result += '---\n\n';
+  }
+  
+  const descriptionLine = includeMetadata && description ? `\n\n> ${description}\n` : '\n';
+  
+  result += `# ${title}${descriptionLine}
+${content}`.trim() + '\n';
+
+  return result;
 }