docusaurus-plugin-llms 0.1.1 → 0.1.3

package/src/index.ts

@@ -8,221 +8,10 @@
  * The plugin runs during the Docusaurus build process and scans all Markdown files in the docs directory.
  */
 
-import * as fs from 'fs/promises';
 import * as path from 'path';
-import matter from 'gray-matter';
-import { minimatch } from 'minimatch';
 import type { LoadContext, Plugin } from '@docusaurus/types';
-
-/**
- * Interface for processed document information
- */
-interface DocInfo {
-  title: string;
-  path: string;
-  url: string;
-  content: string;
-  description: string;
-}
-
-/**
- * Plugin options interface
- */
-interface PluginOptions {
-  /** Whether to generate the llms.txt file (default: true) */
-  generateLLMsTxt?: boolean;
-  
-  /** Whether to generate the llms-full.txt file (default: true) */
-  generateLLMsFullTxt?: boolean;
-  
-  /** Base directory for documentation files (default: 'docs') */
-  docsDir?: string;
-  
-  /** Array of glob patterns for files to ignore */
-  ignoreFiles?: string[];
-  
-  /** Custom title to use in generated files (defaults to site title) */
-  title?: string;
-  
-  /** Custom description to use in generated files (defaults to site tagline) */
-  description?: string;
-  
-  /** Custom file name for the links file (default: 'llms.txt') */
-  llmsTxtFilename?: string;
-  
-  /** Custom file name for the full content file (default: 'llms-full.txt') */
-  llmsFullTxtFilename?: string;
-  
-  /** Whether to include blog content (default: false) */
-  includeBlog?: boolean;
-}
-
-/**
- * Write content to a file
- * @param filePath - Path to write the file to
- * @param data - Content to write
- */
-async function writeFile(filePath: string, data: string): Promise<void> {
-  return fs.writeFile(filePath, data, 'utf8');
-}
-
-/**
- * Read content from a file
- * @param filePath - Path of the file to read
- * @returns Content of the file
- */
-async function readFile(filePath: string): Promise<string> {
-  return fs.readFile(filePath, 'utf8');
-}
-
-/**
- * Check if a file should be ignored based on glob patterns
- * @param filePath - Path to the file
- * @param baseDir - Base directory for relative paths
- * @param ignorePatterns - Glob patterns for files to ignore
- * @returns Whether the file should be ignored
- */
-function shouldIgnoreFile(filePath: string, baseDir: string, ignorePatterns: string[]): boolean {
-  if (ignorePatterns.length === 0) {
-    return false;
-  }
-  
-  const relativePath = path.relative(baseDir, filePath);
-  
-  return ignorePatterns.some(pattern => 
-    minimatch(relativePath, pattern, { matchBase: true })
-  );
-}
-
-/**
- * Recursively reads all Markdown files in a directory
- * @param dir - Directory to scan
- * @param baseDir - Base directory for relative paths
- * @param ignorePatterns - Glob patterns for files to ignore
- * @returns Array of file paths
- */
-async function readMarkdownFiles(dir: string, baseDir: string, ignorePatterns: string[] = []): Promise<string[]> {
-  const files: string[] = [];
-  const entries = await fs.readdir(dir, { withFileTypes: true });
-
-  for (const entry of entries) {
-    const fullPath = path.join(dir, entry.name);
-    
-    if (shouldIgnoreFile(fullPath, baseDir, ignorePatterns)) {
-      continue;
-    }
-    
-    if (entry.isDirectory()) {
-      const subDirFiles = await readMarkdownFiles(fullPath, baseDir, ignorePatterns);
-      files.push(...subDirFiles);
-    } else if (entry.name.endsWith('.md') || entry.name.endsWith('.mdx')) {
-      files.push(fullPath);
-    }
-  }
-
-  return files;
-}
-
-/**
- * Extract title from content or use the filename
- * @param data - Frontmatter data
- * @param content - Markdown content
- * @param filePath - Path to the file
- * @returns Extracted title
- */
-function extractTitle(data: any, content: string, filePath: string): string {
-  // First try frontmatter
-  if (data.title) {
-    return data.title;
-  }
-  
-  // Then try first heading
-  const headingMatch = content.match(/^#\s+(.*)/m);
-  if (headingMatch) {
-    return headingMatch[1].trim();
-  }
-  
-  // Finally use filename
-  return path.basename(filePath, path.extname(filePath))
-    .replace(/-/g, ' ')
-    .replace(/\b\w/g, c => c.toUpperCase());
-}
-
-/**
- * Clean markdown content for LLM consumption
- * @param content - Raw markdown content
- * @returns Cleaned content
- */
-function cleanMarkdownContent(content: string): string {
-  // Remove HTML tags
-  let cleaned = content.replace(/<[^>]*>/g, '');
-  
-  // Normalize whitespace
-  cleaned = cleaned.replace(/\r\n/g, '\n')
-    .replace(/\n{3,}/g, '\n\n')
-    .trim();
-    
-  return cleaned;
-}
-
-/**
- * Process a markdown file and extract its metadata and content
- * @param filePath - Path to the markdown file
- * @param baseDir - Base directory
- * @param siteUrl - Base URL of the site
- * @param pathPrefix - Path prefix for URLs (e.g., 'docs' or 'blog')
- * @returns Processed file data
- */
-async function processMarkdownFile(
-  filePath: string, 
-  baseDir: string, 
-  siteUrl: string,
-  pathPrefix: string = 'docs'
-): Promise<DocInfo> {
-  const content = await readFile(filePath);
-  const { data, content: markdownContent } = matter(content);
-  
-  const relativePath = path.relative(baseDir, filePath);
-  // Convert to URL path format (replace backslashes with forward slashes on Windows)
-  const normalizedPath = relativePath.replace(/\\/g, '/');
-  
-  // Convert .md extension to appropriate path
-  const linkPathBase = normalizedPath.replace(/\.mdx?$/, '');
-  
-  // Handle index files specially
-  const linkPath = linkPathBase.endsWith('index') 
-    ? linkPathBase.replace(/\/index$/, '') 
-    : linkPathBase;
-  
-  // Generate full URL
-  const fullUrl = new URL(`${pathPrefix}/${linkPath}`, siteUrl).toString();
-  
-  // Extract title
-  const title = extractTitle(data, markdownContent, filePath);
-  
-  // Get description from frontmatter or first paragraph
-  let description = data.description || '';
-  if (!description) {
-    const paragraphs = markdownContent.split('\n\n');
-    for (const para of paragraphs) {
-      if (para.trim() && !para.startsWith('#')) {
-        description = para.trim();
-        break;
-      }
-    }
-  }
-  
-  // Clean and process content
-  const cleanedContent = cleanMarkdownContent(markdownContent);
-  
-  return {
-    title,
-    path: normalizedPath,
-    url: fullUrl,
-    content: cleanedContent,
-    description: description || '',
-  };
-}
+import { PluginOptions, PluginContext } from './types';
+import { collectDocFiles, generateStandardLLMFiles, generateCustomLLMFiles } from './generator';
 
 /**
  * A Docusaurus plugin to generate LLM-friendly documentation following
@@ -247,6 +36,10 @@ export default function docusaurusPluginLLMs(
     llmsTxtFilename = 'llms.txt',
     llmsFullTxtFilename = 'llms-full.txt',
     includeBlog = false,
+    pathTransformation,
+    includeOrder = [],
+    includeUnmatchedLast = true,
+    customLLMFiles = [],
   } = options;
 
   const {
@@ -254,6 +47,38 @@ export default function docusaurusPluginLLMs(
     siteConfig,
     outDir,
   } = context;
+  
+  // Build the site URL with proper trailing slash
+  const siteUrl = siteConfig.url + (
+    siteConfig.baseUrl.endsWith('/') 
+      ? siteConfig.baseUrl.slice(0, -1) 
+      : siteConfig.baseUrl || ''
+  );
+  
+  // Create a plugin context object with processed options
+  const pluginContext: PluginContext = {
+    siteDir,
+    outDir,
+    siteUrl,
+    docsDir,
+    docTitle: title || siteConfig.title,
+    docDescription: description || siteConfig.tagline || '',
+    options: {
+      generateLLMsTxt,
+      generateLLMsFullTxt,
+      docsDir,
+      ignoreFiles,
+      title,
+      description,
+      llmsTxtFilename,
+      llmsFullTxtFilename,
+      includeBlog,
+      pathTransformation,
+      includeOrder,
+      includeUnmatchedLast,
+      customLLMFiles,
+    }
+  };
 
   return {
     name: 'docusaurus-plugin-llms',
@@ -263,150 +88,25 @@ export default function docusaurusPluginLLMs(
      */
     async postBuild(): Promise<void> {
       console.log('Generating LLM-friendly documentation...');
-
-      // Custom title and description or fallback to site values
-      const docTitle = title || siteConfig.title;
-      const docDescription = description || siteConfig.tagline || '';
-      
-      // Build the site URL with proper trailing slash
-      const siteUrl = siteConfig.url + (
-        siteConfig.baseUrl.endsWith('/') 
-          ? siteConfig.baseUrl.slice(0, -1) 
-          : siteConfig.baseUrl || ''
-      );
-      
-      // Initialize docs collection
-      const allDocs: DocInfo[] = [];
-
+     
       try {
-        // Process docs directory
-        const fullDocsDir = path.join(siteDir, docsDir);
-        
-        try {
-          await fs.access(fullDocsDir);
-          
-          // Collect all markdown files from docs directory
-          const docFiles = await readMarkdownFiles(fullDocsDir, siteDir, ignoreFiles);
-          
-          if (docFiles.length > 0) {
-            // Process each file
-            for (const filePath of docFiles) {
-              try {
-                const docInfo = await processMarkdownFile(
-                  filePath, 
-                  fullDocsDir, 
-                  siteUrl,
-                  'docs'
-                );
-                allDocs.push(docInfo);
-              } catch (err: any) {
-                console.warn(`Error processing ${filePath}: ${err.message}`);
-              }
-            }
-            console.log(`Processed ${docFiles.length} documentation files`);
-          } else {
-            console.warn('No markdown files found in docs directory.');
-          }
-        } catch (err) {
-          console.warn(`Docs directory not found: ${fullDocsDir}`);
-        }
-        
-        // Process blog if enabled
-        if (includeBlog) {
-          const blogDir = path.join(siteDir, 'blog');
-          
-          try {
-            await fs.access(blogDir);
-            
-            // Collect all markdown files from blog directory
-            const blogFiles = await readMarkdownFiles(blogDir, siteDir, ignoreFiles);
-            
-            if (blogFiles.length > 0) {
-              // Process each file
-              for (const filePath of blogFiles) {
-                try {
-                  const docInfo = await processMarkdownFile(
-                    filePath, 
-                    blogDir, 
-                    siteUrl,
-                    'blog'
-                  );
-                  allDocs.push(docInfo);
-                } catch (err: any) {
-                  console.warn(`Error processing ${filePath}: ${err.message}`);
-                }
-              }
-              console.log(`Processed ${blogFiles.length} blog files`);
-            } else {
-              console.warn('No markdown files found in blog directory.');
-            }
-          } catch (err) {
-            console.warn(`Blog directory not found: ${blogDir}`);
-          }
-        }
+        // Collect all document files
+        const allDocFiles = await collectDocFiles(pluginContext);
         
         // Skip further processing if no documents were found
-        if (allDocs.length === 0) {
+        if (allDocFiles.length === 0) {
           console.warn('No documents found to process.');
           return;
         }
         
-        // Sort files to ensure consistent ordering
-        allDocs.sort((a, b) => a.title.localeCompare(b.title));
-
-        // Generate llms.txt
-        if (generateLLMsTxt) {
-          const llmsTxtPath = path.join(outDir, llmsTxtFilename);
-          const tocItems = allDocs.map(doc => {
-            return `- [${doc.title}](${doc.url})${doc.description ? `: ${doc.description.split('\n')[0]}` : ''}`;
-          });
-
-          const llmsTxtContent = `# ${docTitle}
-
-> ${docDescription}
-
-This file contains links to all documentation sections following the llmtxt.org standard.
-
-## Table of Contents
-
-${tocItems.join('\n')}
-`;
-
-          await writeFile(llmsTxtPath, llmsTxtContent);
-          console.log(`Generated ${llmsTxtFilename}: ${llmsTxtPath}`);
-        }
-
-        // Generate llms-full.txt with all content
-        if (generateLLMsFullTxt) {
-          const llmsFullTxtPath = path.join(outDir, llmsFullTxtFilename);
-          
-          const fullContentSections = allDocs.map(doc => {
-            return `## ${doc.title}
-
-${doc.content}`;
-          });
-
-          const llmsFullTxtContent = `# ${docTitle}
-
-> ${docDescription}
-
-This file contains all documentation content in a single document following the llmtxt.org standard.
-
-${fullContentSections.join('\n\n---\n\n')}
-`;
-
-          await writeFile(llmsFullTxtPath, llmsFullTxtContent);
-          console.log(`Generated ${llmsFullTxtFilename}: ${llmsFullTxtPath}`);
-        }
+        // Process standard LLM files (llms.txt and llms-full.txt)
+        await generateStandardLLMFiles(pluginContext, allDocFiles);
         
-        // Output statistics
-        const stats = {
-          totalDocuments: allDocs.length,
-          totalBytes: allDocs.reduce((sum, doc) => sum + doc.content.length, 0),
-          approxTokens: Math.round(allDocs.reduce((sum, doc) => sum + doc.content.length, 0) / 4), // Rough token estimate
-        };
+        // Process custom LLM files
+        await generateCustomLLMFiles(pluginContext, allDocFiles);
         
-        console.log(`Stats: ${stats.totalDocuments} documents, ${Math.round(stats.totalBytes / 1024)}KB, ~${stats.approxTokens} tokens`);
+        // Output overall statistics
+        console.log(`Stats: ${allDocFiles.length} total available documents processed`);
       } catch (err: any) {
         console.error('Error generating LLM documentation:', err);
       }

package/src/processor.ts

@@ -0,0 +1,236 @@
+/**
+ * Document processing functions for the docusaurus-plugin-llms plugin
+ */
+
+import * as path from 'path';
+import matter from 'gray-matter';
+import { minimatch } from 'minimatch';
+import { DocInfo, PluginContext } from './types';
+import { 
+  readFile, 
+  extractTitle, 
+  cleanMarkdownContent, 
+  applyPathTransformations 
+} from './utils';
+
+/**
+ * Process a markdown file and extract its metadata and content
+ * @param filePath - Path to the markdown file
+ * @param baseDir - Base directory
+ * @param siteUrl - Base URL of the site
+ * @param pathPrefix - Path prefix for URLs (e.g., 'docs' or 'blog')
+ * @param pathTransformation - Path transformation configuration
+ * @returns Processed file data
+ */
+export async function processMarkdownFile(
+  filePath: string, 
+  baseDir: string, 
+  siteUrl: string,
+  pathPrefix: string = 'docs',
+  pathTransformation?: {
+    ignorePaths?: string[];
+    addPaths?: string[];
+  }
+): Promise<DocInfo> {
+  const content = await readFile(filePath);
+  const { data, content: markdownContent } = matter(content);
+  
+  const relativePath = path.relative(baseDir, filePath);
+  // Convert to URL path format (replace backslashes with forward slashes on Windows)
+  const normalizedPath = relativePath.replace(/\\/g, '/');
+  
+  // Convert .md extension to appropriate path
+  const linkPathBase = normalizedPath.replace(/\.mdx?$/, '');
+  
+  // Handle index files specially
+  const linkPath = linkPathBase.endsWith('index') 
+    ? linkPathBase.replace(/\/index$/, '') 
+    : linkPathBase;
+  
+  // Apply path transformations to the link path
+  const transformedLinkPath = applyPathTransformations(linkPath, pathTransformation);
+  
+  // Also apply path transformations to the pathPrefix if it's not empty
+  // This allows removing 'docs' from the path when specified in ignorePaths
+  let transformedPathPrefix = pathPrefix;
+  if (pathPrefix && pathTransformation?.ignorePaths?.includes(pathPrefix)) {
+    transformedPathPrefix = '';
+  }
+  
+  // Generate full URL with transformed path and path prefix
+  const fullUrl = new URL(
+    `${transformedPathPrefix ? `${transformedPathPrefix}/` : ''}${transformedLinkPath}`, 
+    siteUrl
+  ).toString();
+  
+  // Extract title
+  const title = extractTitle(data, markdownContent, filePath);
+  
+  // Get description from frontmatter or first paragraph
+  let description = '';
+  
+  // First priority: Use frontmatter description if available
+  if (data.description) {
+    description = data.description;
+  } else {
+    // Second priority: Find the first non-heading paragraph
+    const paragraphs = markdownContent.split('\n\n');
+    for (const para of paragraphs) {
+      const trimmedPara = para.trim();
+      // Skip empty paragraphs and headings
+      if (trimmedPara && !trimmedPara.startsWith('#')) {
+        description = trimmedPara;
+        break;
+      }
+    }
+    
+    // Third priority: If still no description, use the first heading's content
+    if (!description) {
+      const firstHeadingMatch = markdownContent.match(/^#\s+(.*?)$/m);
+      if (firstHeadingMatch && firstHeadingMatch[1]) {
+        description = firstHeadingMatch[1].trim();
+      }
+    }
+  }
+  
+  // Only remove heading markers at the beginning of descriptions or lines
+  // This preserves # characters that are part of the content
+  if (description) {
+    // Original approach had issues with hashtags inside content
+    // Fix: Only remove # symbols at the beginning of lines or description
+    // that are followed by a space (actual heading markers)
+    description = description.replace(/^(#+)\s+/gm, '');
+    
+    // Special handling for description frontmatter with heading markers
+    if (data.description && data.description.startsWith('#')) {
+      // If the description in frontmatter starts with a heading marker,
+      // we should preserve it in the extracted description
+      description = description.replace(/^#+\s+/, '');
+    }
+    
+    // Preserve inline hashtags (not heading markers)
+    // We don't want to treat hashtags in the middle of content as headings
+    
+    // Validate that the description doesn't contain markdown headings
+    if (description.match(/^#+\s+/m)) {
+      console.warn(`Warning: Description for "${title}" may still contain heading markers`);
+    }
+    
+    // Warn if the description contains HTML tags
+    if (/<[^>]+>/g.test(description)) {
+      console.warn(`Warning: Description for "${title}" contains HTML tags`);
+    }
+    
+    // Warn if the description is very long
+    if (description.length > 500) {
+      console.warn(`Warning: Description for "${title}" is very long (${description.length} characters)`);
+    }
+  }
+  
+  // Clean and process content
+  const cleanedContent = cleanMarkdownContent(markdownContent);
+  
+  return {
+    title,
+    path: normalizedPath,
+    url: fullUrl,
+    content: cleanedContent,
+    description: description || '',
+  };
+}
+
+/**
+ * Process files based on include patterns, ignore patterns, and ordering
+ * @param context - Plugin context
+ * @param allFiles - All available files
+ * @param includePatterns - Patterns for files to include
+ * @param ignorePatterns - Patterns for files to ignore
+ * @param orderPatterns - Patterns for ordering files
+ * @param includeUnmatched - Whether to include unmatched files
+ * @returns Processed files
+ */
+export async function processFilesWithPatterns(
+  context: PluginContext,
+  allFiles: string[],
+  includePatterns: string[] = [],
+  ignorePatterns: string[] = [],
+  orderPatterns: string[] = [],
+  includeUnmatched: boolean = false
+): Promise<DocInfo[]> {
+  const { siteDir, siteUrl, docsDir } = context;
+  
+  // Filter files based on include patterns
+  let filteredFiles = allFiles;
+  
+  if (includePatterns.length > 0) {
+    filteredFiles = allFiles.filter(file => {
+      const relativePath = path.relative(siteDir, file);
+      return includePatterns.some(pattern => 
+        minimatch(relativePath, pattern, { matchBase: true })
+      );
+    });
+  }
+  
+  // Apply ignore patterns
+  if (ignorePatterns.length > 0) {
+    filteredFiles = filteredFiles.filter(file => {
+      const relativePath = path.relative(siteDir, file);
+      return !ignorePatterns.some(pattern => 
+        minimatch(relativePath, pattern, { matchBase: true })
+      );
+    });
+  }
+  
+  // Order files according to orderPatterns
+  let filesToProcess: string[] = [];
+  
+  if (orderPatterns.length > 0) {
+    const matchedFiles = new Set<string>();
+    
+    // Process files according to orderPatterns
+    for (const pattern of orderPatterns) {
+      const matchingFiles = filteredFiles.filter(file => {
+        const relativePath = path.relative(siteDir, file);
+        return minimatch(relativePath, pattern, { matchBase: true }) && !matchedFiles.has(file);
+      });
+      
+      for (const file of matchingFiles) {
+        filesToProcess.push(file);
+        matchedFiles.add(file);
+      }
+    }
+    
+    // Add remaining files if includeUnmatched is true
+    if (includeUnmatched) {
+      const remainingFiles = filteredFiles.filter(file => !matchedFiles.has(file));
+      filesToProcess.push(...remainingFiles);
+    }
+  } else {
+    filesToProcess = filteredFiles;
+  }
+  
+  // Process each file to generate DocInfo
+  const processedDocs: DocInfo[] = [];
+  
+  for (const filePath of filesToProcess) {
+    try {
+      // Determine if this is a blog or docs file
+      const isBlogFile = filePath.includes(path.join(siteDir, 'blog'));
+      const baseDir = isBlogFile ? path.join(siteDir, 'blog') : path.join(siteDir, docsDir);
+      const pathPrefix = isBlogFile ? 'blog' : 'docs';
+      
+      const docInfo = await processMarkdownFile(
+        filePath, 
+        baseDir, 
+        siteUrl,
+        pathPrefix,
+        context.options.pathTransformation
+      );
+      processedDocs.push(docInfo);
+    } catch (err: any) {
+      console.warn(`Error processing ${filePath}: ${err.message}`);
+    }
+  }
+  
+  return processedDocs;
+}