docusaurus-plugin-llms 0.1.1 → 0.1.3

package/src/types.ts

@@ -0,0 +1,113 @@
+/**
+ * Type definitions for the docusaurus-plugin-llms plugin
+ */
+
+import type { LoadContext } from '@docusaurus/types';
+
+/**
+ * Interface for processed document information
+ */
+export interface DocInfo {
+  title: string;
+  path: string;
+  url: string;
+  content: string;
+  description: string;
+}
+
+/**
+ * Interface for custom LLM file configuration
+ */
+export interface CustomLLMFile {
+  /** Name of the output file (e.g., 'llms-python.txt') */
+  filename: string;
+  
+  /** Glob patterns for files to include */
+  includePatterns: string[];
+  
+  /** Whether to include full content (true) or just links (false) */
+  fullContent: boolean;
+  
+  /** Custom title for this file (defaults to site title) */
+  title?: string;
+  
+  /** Custom description for this file (defaults to site description) */
+  description?: string;
+  
+  /** Additional patterns to exclude (combined with global ignoreFiles) */
+  ignorePatterns?: string[];
+  
+  /** Order patterns for controlling file ordering (similar to includeOrder) */
+  orderPatterns?: string[];
+  
+  /** Whether to include unmatched files last (default: false) */
+  includeUnmatchedLast?: boolean;
+  
+  /** Version information for this LLM file */
+  version?: string;
+}
+
+/**
+ * Plugin options interface
+ */
+export interface PluginOptions {
+  /** Whether to generate the llms.txt file (default: true) */
+  generateLLMsTxt?: boolean;
+  
+  /** Whether to generate the llms-full.txt file (default: true) */
+  generateLLMsFullTxt?: boolean;
+  
+  /** Base directory for documentation files (default: 'docs') */
+  docsDir?: string;
+  
+  /** Array of glob patterns for files to ignore */
+  ignoreFiles?: string[];
+  
+  /** Custom title to use in generated files (defaults to site title) */
+  title?: string;
+  
+  /** Custom description to use in generated files (defaults to site tagline) */
+  description?: string;
+  
+  /** Custom file name for the links file (default: 'llms.txt') */
+  llmsTxtFilename?: string;
+  
+  /** Custom file name for the full content file (default: 'llms-full.txt') */
+  llmsFullTxtFilename?: string;
+  
+  /** Whether to include blog content (default: false) */
+  includeBlog?: boolean;
+  
+  /** Path transformation options for URL construction */
+  pathTransformation?: {
+    /** Path segments to ignore when constructing URLs (will be removed if found) */
+    ignorePaths?: string[];
+    /** Path segments to add when constructing URLs (will be prepended if not already present) */
+    addPaths?: string[];
+  };
+
+  /** Array of glob patterns for controlling the order of files (files will be processed in the order of patterns) */
+  includeOrder?: string[];
+  
+  /** Whether to include files that don't match any pattern in includeOrder at the end (default: true) */
+  includeUnmatchedLast?: boolean;
+  
+  /** Array of custom LLM file configurations */
+  customLLMFiles?: CustomLLMFile[];
+  
+  /** Global version for all generated LLM files */
+  version?: string;
+}
+
+/**
+ * Plugin context with processed options
+ */
+export interface PluginContext {
+  siteDir: string;
+  outDir: string;
+  siteUrl: string;
+  docsDir: string;
+  docTitle: string;
+  docDescription: string;
+  options: PluginOptions;
+} 

package/src/utils.ts

@@ -0,0 +1,165 @@
+/**
+ * Utility functions for the docusaurus-plugin-llms plugin
+ */
+
+import * as fs from 'fs/promises';
+import * as path from 'path';
+import { minimatch } from 'minimatch';
+import { PluginOptions } from './types';
+
+/**
+ * Write content to a file
+ * @param filePath - Path to write the file to
+ * @param data - Content to write
+ */
+export async function writeFile(filePath: string, data: string): Promise<void> {
+  return fs.writeFile(filePath, data, 'utf8');
+}
+
+/**
+ * Read content from a file
+ * @param filePath - Path of the file to read
+ * @returns Content of the file
+ */
+export async function readFile(filePath: string): Promise<string> {
+  return fs.readFile(filePath, 'utf8');
+}
+
+/**
+ * Check if a file should be ignored based on glob patterns
+ * @param filePath - Path to the file
+ * @param baseDir - Base directory for relative paths
+ * @param ignorePatterns - Glob patterns for files to ignore
+ * @returns Whether the file should be ignored
+ */
+export function shouldIgnoreFile(filePath: string, baseDir: string, ignorePatterns: string[]): boolean {
+  if (ignorePatterns.length === 0) {
+    return false;
+  }
+  
+  const relativePath = path.relative(baseDir, filePath);
+  
+  return ignorePatterns.some(pattern => 
+    minimatch(relativePath, pattern, { matchBase: true })
+  );
+}
+
+/**
+ * Recursively reads all Markdown files in a directory
+ * @param dir - Directory to scan
+ * @param baseDir - Base directory for relative paths
+ * @param ignorePatterns - Glob patterns for files to ignore
+ * @returns Array of file paths
+ */
+export async function readMarkdownFiles(dir: string, baseDir: string, ignorePatterns: string[] = []): Promise<string[]> {
+  const files: string[] = [];
+  const entries = await fs.readdir(dir, { withFileTypes: true });
+
+  for (const entry of entries) {
+    const fullPath = path.join(dir, entry.name);
+    
+    if (shouldIgnoreFile(fullPath, baseDir, ignorePatterns)) {
+      continue;
+    }
+    
+    if (entry.isDirectory()) {
+      const subDirFiles = await readMarkdownFiles(fullPath, baseDir, ignorePatterns);
+      files.push(...subDirFiles);
+    } else if (entry.name.endsWith('.md') || entry.name.endsWith('.mdx')) {
+      files.push(fullPath);
+    }
+  }
+
+  return files;
+}
+
+/**
+ * Extract title from content or use the filename
+ * @param data - Frontmatter data
+ * @param content - Markdown content
+ * @param filePath - Path to the file
+ * @returns Extracted title
+ */
+export function extractTitle(data: any, content: string, filePath: string): string {
+  // First try frontmatter
+  if (data.title) {
+    return data.title;
+  }
+  
+  // Then try first heading
+  const headingMatch = content.match(/^#\s+(.*)/m);
+  if (headingMatch) {
+    return headingMatch[1].trim();
+  }
+  
+  // Finally use filename
+  return path.basename(filePath, path.extname(filePath))
+    .replace(/-/g, ' ')
+    .replace(/\b\w/g, c => c.toUpperCase());
+}
+
+/**
+ * Clean markdown content for LLM consumption
+ * @param content - Raw markdown content
+ * @returns Cleaned content
+ */
+export function cleanMarkdownContent(content: string): string {
+  // Remove HTML tags
+  let cleaned = content.replace(/<[^>]*>/g, '');
+  
+  // Normalize whitespace
+  cleaned = cleaned.replace(/\r\n/g, '\n')
+    .replace(/\n{3,}/g, '\n\n')
+    .trim();
+    
+  return cleaned;
+}
+
+/**
+ * Apply path transformations according to configuration
+ * @param urlPath - Original URL path
+ * @param pathTransformation - Path transformation configuration
+ * @returns Transformed URL path
+ */
+export function applyPathTransformations(
+  urlPath: string,
+  pathTransformation?: PluginOptions['pathTransformation']
+): string {
+  if (!pathTransformation) {
+    return urlPath;
+  }
+
+  let transformedPath = urlPath;
+  
+  // Remove ignored path segments
+  if (pathTransformation.ignorePaths?.length) {
+    for (const ignorePath of pathTransformation.ignorePaths) {
+      // Create a regex that matches the ignore path at the beginning, middle, or end of the path
+      // We use word boundaries to ensure we match complete path segments
+      const ignoreRegex = new RegExp(`(^|/)(${ignorePath})(/|$)`, 'g');
+      transformedPath = transformedPath.replace(ignoreRegex, '$1$3');
+    }
+    
+    // Clean up any double slashes that might have been created
+    transformedPath = transformedPath.replace(/\/+/g, '/');
+    
+    // Remove leading slash if present
+    transformedPath = transformedPath.replace(/^\//, '');
+  }
+  
+  // Add path segments if they're not already present
+  if (pathTransformation.addPaths?.length) {
+    // Process in reverse order to maintain the specified order in the final path
+    // This is because each path is prepended to the front
+    const pathsToAdd = [...pathTransformation.addPaths].reverse();
+    
+    for (const addPath of pathsToAdd) {
+      // Only add if not already present at the beginning
+      if (!transformedPath.startsWith(addPath + '/') && transformedPath !== addPath) {
+        transformedPath = `${addPath}/${transformedPath}`;
+      }
+    }
+  }
+  
+  return transformedPath;
+}