docusaurus-plugin-llms 0.1.2 → 0.1.3

package/src/processor.ts

@@ -0,0 +1,236 @@
+/**
+ * Document processing functions for the docusaurus-plugin-llms plugin
+ */
+
+import * as path from 'path';
+import matter from 'gray-matter';
+import { minimatch } from 'minimatch';
+import { DocInfo, PluginContext } from './types';
+import { 
+  readFile, 
+  extractTitle, 
+  cleanMarkdownContent, 
+  applyPathTransformations 
+} from './utils';
+
+/**
+ * Process a markdown file and extract its metadata and content
+ * @param filePath - Path to the markdown file
+ * @param baseDir - Base directory
+ * @param siteUrl - Base URL of the site
+ * @param pathPrefix - Path prefix for URLs (e.g., 'docs' or 'blog')
+ * @param pathTransformation - Path transformation configuration
+ * @returns Processed file data
+ */
+export async function processMarkdownFile(
+  filePath: string, 
+  baseDir: string, 
+  siteUrl: string,
+  pathPrefix: string = 'docs',
+  pathTransformation?: {
+    ignorePaths?: string[];
+    addPaths?: string[];
+  }
+): Promise<DocInfo> {
+  const content = await readFile(filePath);
+  const { data, content: markdownContent } = matter(content);
+  
+  const relativePath = path.relative(baseDir, filePath);
+  // Convert to URL path format (replace backslashes with forward slashes on Windows)
+  const normalizedPath = relativePath.replace(/\\/g, '/');
+  
+  // Convert .md extension to appropriate path
+  const linkPathBase = normalizedPath.replace(/\.mdx?$/, '');
+  
+  // Handle index files specially
+  const linkPath = linkPathBase.endsWith('index') 
+    ? linkPathBase.replace(/\/index$/, '') 
+    : linkPathBase;
+  
+  // Apply path transformations to the link path
+  const transformedLinkPath = applyPathTransformations(linkPath, pathTransformation);
+  
+  // Also apply path transformations to the pathPrefix if it's not empty
+  // This allows removing 'docs' from the path when specified in ignorePaths
+  let transformedPathPrefix = pathPrefix;
+  if (pathPrefix && pathTransformation?.ignorePaths?.includes(pathPrefix)) {
+    transformedPathPrefix = '';
+  }
+  
+  // Generate full URL with transformed path and path prefix
+  const fullUrl = new URL(
+    `${transformedPathPrefix ? `${transformedPathPrefix}/` : ''}${transformedLinkPath}`, 
+    siteUrl
+  ).toString();
+  
+  // Extract title
+  const title = extractTitle(data, markdownContent, filePath);
+  
+  // Get description from frontmatter or first paragraph
+  let description = '';
+  
+  // First priority: Use frontmatter description if available
+  if (data.description) {
+    description = data.description;
+  } else {
+    // Second priority: Find the first non-heading paragraph
+    const paragraphs = markdownContent.split('\n\n');
+    for (const para of paragraphs) {
+      const trimmedPara = para.trim();
+      // Skip empty paragraphs and headings
+      if (trimmedPara && !trimmedPara.startsWith('#')) {
+        description = trimmedPara;
+        break;
+      }
+    }
+    
+    // Third priority: If still no description, use the first heading's content
+    if (!description) {
+      const firstHeadingMatch = markdownContent.match(/^#\s+(.*?)$/m);
+      if (firstHeadingMatch && firstHeadingMatch[1]) {
+        description = firstHeadingMatch[1].trim();
+      }
+    }
+  }
+  
+  // Only remove heading markers at the beginning of descriptions or lines
+  // This preserves # characters that are part of the content
+  if (description) {
+    // Original approach had issues with hashtags inside content
+    // Fix: Only remove # symbols at the beginning of lines or description
+    // that are followed by a space (actual heading markers)
+    description = description.replace(/^(#+)\s+/gm, '');
+    
+    // Special handling for description frontmatter with heading markers
+    if (data.description && data.description.startsWith('#')) {
+      // If the description in frontmatter starts with a heading marker,
+      // we should preserve it in the extracted description
+      description = description.replace(/^#+\s+/, '');
+    }
+    
+    // Preserve inline hashtags (not heading markers)
+    // We don't want to treat hashtags in the middle of content as headings
+    
+    // Validate that the description doesn't contain markdown headings
+    if (description.match(/^#+\s+/m)) {
+      console.warn(`Warning: Description for "${title}" may still contain heading markers`);
+    }
+    
+    // Warn if the description contains HTML tags
+    if (/<[^>]+>/g.test(description)) {
+      console.warn(`Warning: Description for "${title}" contains HTML tags`);
+    }
+    
+    // Warn if the description is very long
+    if (description.length > 500) {
+      console.warn(`Warning: Description for "${title}" is very long (${description.length} characters)`);
+    }
+  }
+  
+  // Clean and process content
+  const cleanedContent = cleanMarkdownContent(markdownContent);
+  
+  return {
+    title,
+    path: normalizedPath,
+    url: fullUrl,
+    content: cleanedContent,
+    description: description || '',
+  };
+}
+
+/**
+ * Process files based on include patterns, ignore patterns, and ordering
+ * @param context - Plugin context
+ * @param allFiles - All available files
+ * @param includePatterns - Patterns for files to include
+ * @param ignorePatterns - Patterns for files to ignore
+ * @param orderPatterns - Patterns for ordering files
+ * @param includeUnmatched - Whether to include unmatched files
+ * @returns Processed files
+ */
+export async function processFilesWithPatterns(
+  context: PluginContext,
+  allFiles: string[],
+  includePatterns: string[] = [],
+  ignorePatterns: string[] = [],
+  orderPatterns: string[] = [],
+  includeUnmatched: boolean = false
+): Promise<DocInfo[]> {
+  const { siteDir, siteUrl, docsDir } = context;
+  
+  // Filter files based on include patterns
+  let filteredFiles = allFiles;
+  
+  if (includePatterns.length > 0) {
+    filteredFiles = allFiles.filter(file => {
+      const relativePath = path.relative(siteDir, file);
+      return includePatterns.some(pattern => 
+        minimatch(relativePath, pattern, { matchBase: true })
+      );
+    });
+  }
+  
+  // Apply ignore patterns
+  if (ignorePatterns.length > 0) {
+    filteredFiles = filteredFiles.filter(file => {
+      const relativePath = path.relative(siteDir, file);
+      return !ignorePatterns.some(pattern => 
+        minimatch(relativePath, pattern, { matchBase: true })
+      );
+    });
+  }
+  
+  // Order files according to orderPatterns
+  let filesToProcess: string[] = [];
+  
+  if (orderPatterns.length > 0) {
+    const matchedFiles = new Set<string>();
+    
+    // Process files according to orderPatterns
+    for (const pattern of orderPatterns) {
+      const matchingFiles = filteredFiles.filter(file => {
+        const relativePath = path.relative(siteDir, file);
+        return minimatch(relativePath, pattern, { matchBase: true }) && !matchedFiles.has(file);
+      });
+      
+      for (const file of matchingFiles) {
+        filesToProcess.push(file);
+        matchedFiles.add(file);
+      }
+    }
+    
+    // Add remaining files if includeUnmatched is true
+    if (includeUnmatched) {
+      const remainingFiles = filteredFiles.filter(file => !matchedFiles.has(file));
+      filesToProcess.push(...remainingFiles);
+    }
+  } else {
+    filesToProcess = filteredFiles;
+  }
+  
+  // Process each file to generate DocInfo
+  const processedDocs: DocInfo[] = [];
+  
+  for (const filePath of filesToProcess) {
+    try {
+      // Determine if this is a blog or docs file
+      const isBlogFile = filePath.includes(path.join(siteDir, 'blog'));
+      const baseDir = isBlogFile ? path.join(siteDir, 'blog') : path.join(siteDir, docsDir);
+      const pathPrefix = isBlogFile ? 'blog' : 'docs';
+      
+      const docInfo = await processMarkdownFile(
+        filePath, 
+        baseDir, 
+        siteUrl,
+        pathPrefix,
+        context.options.pathTransformation
+      );
+      processedDocs.push(docInfo);
+    } catch (err: any) {
+      console.warn(`Error processing ${filePath}: ${err.message}`);
+    }
+  }
+  
+  return processedDocs;
+} 

package/src/types.ts

@@ -0,0 +1,113 @@
+/**
+ * Type definitions for the docusaurus-plugin-llms plugin
+ */
+
+import type { LoadContext } from '@docusaurus/types';
+
+/**
+ * Interface for processed document information
+ */
+export interface DocInfo {
+  title: string;
+  path: string;
+  url: string;
+  content: string;
+  description: string;
+}
+
+/**
+ * Interface for custom LLM file configuration
+ */
+export interface CustomLLMFile {
+  /** Name of the output file (e.g., 'llms-python.txt') */
+  filename: string;
+  
+  /** Glob patterns for files to include */
+  includePatterns: string[];
+  
+  /** Whether to include full content (true) or just links (false) */
+  fullContent: boolean;
+  
+  /** Custom title for this file (defaults to site title) */
+  title?: string;
+  
+  /** Custom description for this file (defaults to site description) */
+  description?: string;
+  
+  /** Additional patterns to exclude (combined with global ignoreFiles) */
+  ignorePatterns?: string[];
+  
+  /** Order patterns for controlling file ordering (similar to includeOrder) */
+  orderPatterns?: string[];
+  
+  /** Whether to include unmatched files last (default: false) */
+  includeUnmatchedLast?: boolean;
+  
+  /** Version information for this LLM file */
+  version?: string;
+}
+
+/**
+ * Plugin options interface
+ */
+export interface PluginOptions {
+  /** Whether to generate the llms.txt file (default: true) */
+  generateLLMsTxt?: boolean;
+  
+  /** Whether to generate the llms-full.txt file (default: true) */
+  generateLLMsFullTxt?: boolean;
+  
+  /** Base directory for documentation files (default: 'docs') */
+  docsDir?: string;
+  
+  /** Array of glob patterns for files to ignore */
+  ignoreFiles?: string[];
+  
+  /** Custom title to use in generated files (defaults to site title) */
+  title?: string;
+  
+  /** Custom description to use in generated files (defaults to site tagline) */
+  description?: string;
+  
+  /** Custom file name for the links file (default: 'llms.txt') */
+  llmsTxtFilename?: string;
+  
+  /** Custom file name for the full content file (default: 'llms-full.txt') */
+  llmsFullTxtFilename?: string;
+  
+  /** Whether to include blog content (default: false) */
+  includeBlog?: boolean;
+  
+  /** Path transformation options for URL construction */
+  pathTransformation?: {
+    /** Path segments to ignore when constructing URLs (will be removed if found) */
+    ignorePaths?: string[];
+    /** Path segments to add when constructing URLs (will be prepended if not already present) */
+    addPaths?: string[];
+  };
+
+  /** Array of glob patterns for controlling the order of files (files will be processed in the order of patterns) */
+  includeOrder?: string[];
+  
+  /** Whether to include files that don't match any pattern in includeOrder at the end (default: true) */
+  includeUnmatchedLast?: boolean;
+  
+  /** Array of custom LLM file configurations */
+  customLLMFiles?: CustomLLMFile[];
+  
+  /** Global version for all generated LLM files */
+  version?: string;
+}
+
+/**
+ * Plugin context with processed options
+ */
+export interface PluginContext {
+  siteDir: string;
+  outDir: string;
+  siteUrl: string;
+  docsDir: string;
+  docTitle: string;
+  docDescription: string;
+  options: PluginOptions;
+} 

package/src/utils.ts

@@ -0,0 +1,165 @@
+/**
+ * Utility functions for the docusaurus-plugin-llms plugin
+ */
+
+import * as fs from 'fs/promises';
+import * as path from 'path';
+import { minimatch } from 'minimatch';
+import { PluginOptions } from './types';
+
+/**
+ * Write content to a file
+ * @param filePath - Path to write the file to
+ * @param data - Content to write
+ */
+export async function writeFile(filePath: string, data: string): Promise<void> {
+  return fs.writeFile(filePath, data, 'utf8');
+}
+
+/**
+ * Read content from a file
+ * @param filePath - Path of the file to read
+ * @returns Content of the file
+ */
+export async function readFile(filePath: string): Promise<string> {
+  return fs.readFile(filePath, 'utf8');
+}
+
+/**
+ * Check if a file should be ignored based on glob patterns
+ * @param filePath - Path to the file
+ * @param baseDir - Base directory for relative paths
+ * @param ignorePatterns - Glob patterns for files to ignore
+ * @returns Whether the file should be ignored
+ */
+export function shouldIgnoreFile(filePath: string, baseDir: string, ignorePatterns: string[]): boolean {
+  if (ignorePatterns.length === 0) {
+    return false;
+  }
+  
+  const relativePath = path.relative(baseDir, filePath);
+  
+  return ignorePatterns.some(pattern => 
+    minimatch(relativePath, pattern, { matchBase: true })
+  );
+}
+
+/**
+ * Recursively reads all Markdown files in a directory
+ * @param dir - Directory to scan
+ * @param baseDir - Base directory for relative paths
+ * @param ignorePatterns - Glob patterns for files to ignore
+ * @returns Array of file paths
+ */
+export async function readMarkdownFiles(dir: string, baseDir: string, ignorePatterns: string[] = []): Promise<string[]> {
+  const files: string[] = [];
+  const entries = await fs.readdir(dir, { withFileTypes: true });
+
+  for (const entry of entries) {
+    const fullPath = path.join(dir, entry.name);
+    
+    if (shouldIgnoreFile(fullPath, baseDir, ignorePatterns)) {
+      continue;
+    }
+    
+    if (entry.isDirectory()) {
+      const subDirFiles = await readMarkdownFiles(fullPath, baseDir, ignorePatterns);
+      files.push(...subDirFiles);
+    } else if (entry.name.endsWith('.md') || entry.name.endsWith('.mdx')) {
+      files.push(fullPath);
+    }
+  }
+
+  return files;
+}
+
+/**
+ * Extract title from content or use the filename
+ * @param data - Frontmatter data
+ * @param content - Markdown content
+ * @param filePath - Path to the file
+ * @returns Extracted title
+ */
+export function extractTitle(data: any, content: string, filePath: string): string {
+  // First try frontmatter
+  if (data.title) {
+    return data.title;
+  }
+  
+  // Then try first heading
+  const headingMatch = content.match(/^#\s+(.*)/m);
+  if (headingMatch) {
+    return headingMatch[1].trim();
+  }
+  
+  // Finally use filename
+  return path.basename(filePath, path.extname(filePath))
+    .replace(/-/g, ' ')
+    .replace(/\b\w/g, c => c.toUpperCase());
+}
+
+/**
+ * Clean markdown content for LLM consumption
+ * @param content - Raw markdown content
+ * @returns Cleaned content
+ */
+export function cleanMarkdownContent(content: string): string {
+  // Remove HTML tags
+  let cleaned = content.replace(/<[^>]*>/g, '');
+  
+  // Normalize whitespace
+  cleaned = cleaned.replace(/\r\n/g, '\n')
+    .replace(/\n{3,}/g, '\n\n')
+    .trim();
+    
+  return cleaned;
+}
+
+/**
+ * Apply path transformations according to configuration
+ * @param urlPath - Original URL path
+ * @param pathTransformation - Path transformation configuration
+ * @returns Transformed URL path
+ */
+export function applyPathTransformations(
+  urlPath: string,
+  pathTransformation?: PluginOptions['pathTransformation']
+): string {
+  if (!pathTransformation) {
+    return urlPath;
+  }
+
+  let transformedPath = urlPath;
+  
+  // Remove ignored path segments
+  if (pathTransformation.ignorePaths?.length) {
+    for (const ignorePath of pathTransformation.ignorePaths) {
+      // Create a regex that matches the ignore path at the beginning, middle, or end of the path
+      // We use word boundaries to ensure we match complete path segments
+      const ignoreRegex = new RegExp(`(^|/)(${ignorePath})(/|$)`, 'g');
+      transformedPath = transformedPath.replace(ignoreRegex, '$1$3');
+    }
+    
+    // Clean up any double slashes that might have been created
+    transformedPath = transformedPath.replace(/\/+/g, '/');
+    
+    // Remove leading slash if present
+    transformedPath = transformedPath.replace(/^\//, '');
+  }
+  
+  // Add path segments if they're not already present
+  if (pathTransformation.addPaths?.length) {
+    // Process in reverse order to maintain the specified order in the final path
+    // This is because each path is prepended to the front
+    const pathsToAdd = [...pathTransformation.addPaths].reverse();
+    
+    for (const addPath of pathsToAdd) {
+      // Only add if not already present at the beginning
+      if (!transformedPath.startsWith(addPath + '/') && transformedPath !== addPath) {
+        transformedPath = `${addPath}/${transformedPath}`;
+      }
+    }
+  }
+  
+  return transformedPath;
+}