docusaurus-plugin-llms 0.1.1 → 0.1.3

package/lib/utils.js

@@ -0,0 +1,177 @@
+"use strict";
+/**
+ * Utility functions for the docusaurus-plugin-llms plugin
+ */
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.writeFile = writeFile;
+exports.readFile = readFile;
+exports.shouldIgnoreFile = shouldIgnoreFile;
+exports.readMarkdownFiles = readMarkdownFiles;
+exports.extractTitle = extractTitle;
+exports.cleanMarkdownContent = cleanMarkdownContent;
+exports.applyPathTransformations = applyPathTransformations;
+const fs = __importStar(require("fs/promises"));
+const path = __importStar(require("path"));
+const minimatch_1 = require("minimatch");
+/**
+ * Write content to a file
+ * @param filePath - Path to write the file to
+ * @param data - Content to write
+ */
+async function writeFile(filePath, data) {
+    return fs.writeFile(filePath, data, 'utf8');
+}
+/**
+ * Read content from a file
+ * @param filePath - Path of the file to read
+ * @returns Content of the file
+ */
+async function readFile(filePath) {
+    return fs.readFile(filePath, 'utf8');
+}
+/**
+ * Check if a file should be ignored based on glob patterns
+ * @param filePath - Path to the file
+ * @param baseDir - Base directory for relative paths
+ * @param ignorePatterns - Glob patterns for files to ignore
+ * @returns Whether the file should be ignored
+ */
+function shouldIgnoreFile(filePath, baseDir, ignorePatterns) {
+    if (ignorePatterns.length === 0) {
+        return false;
+    }
+    const relativePath = path.relative(baseDir, filePath);
+    return ignorePatterns.some(pattern => (0, minimatch_1.minimatch)(relativePath, pattern, { matchBase: true }));
+}
+/**
+ * Recursively reads all Markdown files in a directory
+ * @param dir - Directory to scan
+ * @param baseDir - Base directory for relative paths
+ * @param ignorePatterns - Glob patterns for files to ignore
+ * @returns Array of file paths
+ */
+async function readMarkdownFiles(dir, baseDir, ignorePatterns = []) {
+    const files = [];
+    const entries = await fs.readdir(dir, { withFileTypes: true });
+    for (const entry of entries) {
+        const fullPath = path.join(dir, entry.name);
+        if (shouldIgnoreFile(fullPath, baseDir, ignorePatterns)) {
+            continue;
+        }
+        if (entry.isDirectory()) {
+            const subDirFiles = await readMarkdownFiles(fullPath, baseDir, ignorePatterns);
+            files.push(...subDirFiles);
+        }
+        else if (entry.name.endsWith('.md') || entry.name.endsWith('.mdx')) {
+            files.push(fullPath);
+        }
+    }
+    return files;
+}
+/**
+ * Extract title from content or use the filename
+ * @param data - Frontmatter data
+ * @param content - Markdown content
+ * @param filePath - Path to the file
+ * @returns Extracted title
+ */
+function extractTitle(data, content, filePath) {
+    // First try frontmatter
+    if (data.title) {
+        return data.title;
+    }
+    // Then try first heading
+    const headingMatch = content.match(/^#\s+(.*)/m);
+    if (headingMatch) {
+        return headingMatch[1].trim();
+    }
+    // Finally use filename
+    return path.basename(filePath, path.extname(filePath))
+        .replace(/-/g, ' ')
+        .replace(/\b\w/g, c => c.toUpperCase());
+}
+/**
+ * Clean markdown content for LLM consumption
+ * @param content - Raw markdown content
+ * @returns Cleaned content
+ */
+function cleanMarkdownContent(content) {
+    // Remove HTML tags
+    let cleaned = content.replace(/<[^>]*>/g, '');
+    // Normalize whitespace
+    cleaned = cleaned.replace(/\r\n/g, '\n')
+        .replace(/\n{3,}/g, '\n\n')
+        .trim();
+    return cleaned;
+}
+/**
+ * Apply path transformations according to configuration
+ * @param urlPath - Original URL path
+ * @param pathTransformation - Path transformation configuration
+ * @returns Transformed URL path
+ */
+function applyPathTransformations(urlPath, pathTransformation) {
+    if (!pathTransformation) {
+        return urlPath;
+    }
+    let transformedPath = urlPath;
+    // Remove ignored path segments
+    if (pathTransformation.ignorePaths?.length) {
+        for (const ignorePath of pathTransformation.ignorePaths) {
+            // Create a regex that matches the ignore path at the beginning, middle, or end of the path
+            // We use word boundaries to ensure we match complete path segments
+            const ignoreRegex = new RegExp(`(^|/)(${ignorePath})(/|$)`, 'g');
+            transformedPath = transformedPath.replace(ignoreRegex, '$1$3');
+        }
+        // Clean up any double slashes that might have been created
+        transformedPath = transformedPath.replace(/\/+/g, '/');
+        // Remove leading slash if present
+        transformedPath = transformedPath.replace(/^\//, '');
+    }
+    // Add path segments if they're not already present
+    if (pathTransformation.addPaths?.length) {
+        // Process in reverse order to maintain the specified order in the final path
+        // This is because each path is prepended to the front
+        const pathsToAdd = [...pathTransformation.addPaths].reverse();
+        for (const addPath of pathsToAdd) {
+            // Only add if not already present at the beginning
+            if (!transformedPath.startsWith(addPath + '/') && transformedPath !== addPath) {
+                transformedPath = `${addPath}/${transformedPath}`;
+            }
+        }
+    }
+    return transformedPath;
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "docusaurus-plugin-llms",
-  "version": "0.1.1",
+  "version": "0.1.3",
   "description": "Docusaurus plugin for generating LLM-friendly documentation following the llmtxt.org standard",
   "main": "lib/index.js",
   "scripts": {
@@ -8,7 +8,9 @@
     "watch": "tsc --watch",
     "cleanup": "node cleanup.js",
     "prepublishOnly": "npm run build && npm run cleanup",
-    "test": "echo \"No tests specified\""
+    "test:unit": "node tests/test-path-transforms.js",
+    "test:integration": "node tests/test-path-transformation.js",
+    "test": "npm run build && npm run test:unit && npm run test:integration"
   },
   "files": [
     "lib",

package/src/generator.ts

@@ -0,0 +1,266 @@
+/**
+ * LLM file generation functions for the docusaurus-plugin-llms plugin
+ */
+
+import * as path from 'path';
+import * as fs from 'fs/promises';
+import { DocInfo, PluginContext, CustomLLMFile } from './types';
+import { writeFile, readMarkdownFiles } from './utils';
+import { processFilesWithPatterns } from './processor';
+
+/**
+ * Clean a description for use in a TOC item
+ * @param description - The original description
+ * @returns Cleaned description suitable for TOC
+ */
+function cleanDescriptionForToc(description: string): string {
+  if (!description) return '';
+  
+  // Get just the first line for TOC display
+  const firstLine = description.split('\n')[0];
+  
+  // Remove heading markers only at the beginning of the line
+  // Be careful to only remove actual heading markers (# followed by space at beginning)
+  // and not hashtag symbols that are part of the content (inline hashtags)
+  const cleaned = firstLine.replace(/^(#+)\s+/g, '');
+  
+  // Truncate if too long (150 characters max with ellipsis)
+  return cleaned.length > 150 ? cleaned.substring(0, 147) + '...' : cleaned;
+}
+
+/**
+ * Generate an LLM-friendly file
+ * @param docs - Processed document information
+ * @param outputPath - Path to write the output file
+ * @param fileTitle - Title for the file
+ * @param fileDescription - Description for the file
+ * @param includeFullContent - Whether to include full content or just links
+ * @param version - Version of the file
+ */
+export async function generateLLMFile(
+  docs: DocInfo[],
+  outputPath: string,
+  fileTitle: string,
+  fileDescription: string,
+  includeFullContent: boolean,
+  version?: string
+): Promise<void> {
+  console.log(`Generating file: ${outputPath}, version: ${version || 'undefined'}`);
+  const versionInfo = version ? `\n\nVersion: ${version}` : '';
+  
+  if (includeFullContent) {
+    // Generate full content file
+    const fullContentSections = docs.map(doc => {
+      return `## ${doc.title}
+
+${doc.content}`;
+    });
+
+    const llmFileContent = `# ${fileTitle}
+
+> ${fileDescription}${versionInfo}
+
+This file contains all documentation content in a single document following the llmtxt.org standard.
+
+${fullContentSections.join('\n\n---\n\n')}
+`;
+
+    await writeFile(outputPath, llmFileContent);
+  } else {
+    // Generate links-only file
+    const tocItems = docs.map(doc => {
+      // Clean and format the description for TOC
+      const cleanedDescription = cleanDescriptionForToc(doc.description);
+      
+      return `- [${doc.title}](${doc.url})${cleanedDescription ? `: ${cleanedDescription}` : ''}`;
+    });
+
+    const llmFileContent = `# ${fileTitle}
+
+> ${fileDescription}${versionInfo}
+
+This file contains links to documentation sections following the llmtxt.org standard.
+
+## Table of Contents
+
+${tocItems.join('\n')}
+`;
+
+    await writeFile(outputPath, llmFileContent);
+  }
+  
+  console.log(`Generated: ${outputPath}`);
+}
+
+/**
+ * Generate standard LLM files (llms.txt and llms-full.txt)
+ * @param context - Plugin context
+ * @param allDocFiles - Array of all document files
+ */
+export async function generateStandardLLMFiles(
+  context: PluginContext,
+  allDocFiles: string[]
+): Promise<void> {
+  const { 
+    outDir, 
+    docTitle, 
+    docDescription, 
+    options 
+  } = context;
+  
+  const { 
+    generateLLMsTxt, 
+    generateLLMsFullTxt,
+    llmsTxtFilename = 'llms.txt',
+    llmsFullTxtFilename = 'llms-full.txt',
+    includeOrder = [],
+    includeUnmatchedLast = true,
+    version
+  } = options;
+  
+  if (!generateLLMsTxt && !generateLLMsFullTxt) {
+    return;
+  }
+  
+  // Process files for the standard outputs
+  const processedDocs = await processFilesWithPatterns(
+    context,
+    allDocFiles,
+    [], // No specific include patterns - include all
+    [], // No additional ignore patterns beyond global ignoreFiles
+    includeOrder,
+    includeUnmatchedLast
+  );
+  
+  console.log(`Processed ${processedDocs.length} documentation files for standard LLM files`);
+  
+  // Generate llms.txt
+  if (generateLLMsTxt) {
+    const llmsTxtPath = path.join(outDir, llmsTxtFilename);
+    await generateLLMFile(
+      processedDocs,
+      llmsTxtPath,
+      docTitle,
+      docDescription,
+      false, // links only
+      version
+    );
+  }
+
+  // Generate llms-full.txt
+  if (generateLLMsFullTxt) {
+    const llmsFullTxtPath = path.join(outDir, llmsFullTxtFilename);
+    await generateLLMFile(
+      processedDocs,
+      llmsFullTxtPath,
+      docTitle,
+      docDescription,
+      true, // full content
+      version
+    );
+  }
+}
+
+/**
+ * Generate custom LLM files based on configuration
+ * @param context - Plugin context
+ * @param allDocFiles - Array of all document files
+ */
+export async function generateCustomLLMFiles(
+  context: PluginContext,
+  allDocFiles: string[]
+): Promise<void> {
+  const { outDir, docTitle, docDescription, options } = context;
+  const { customLLMFiles = [], ignoreFiles = [] } = options;
+  
+  if (customLLMFiles.length === 0) {
+    return;
+  }
+  
+  console.log(`Generating ${customLLMFiles.length} custom LLM files...`);
+  
+  for (const customFile of customLLMFiles) {
+    console.log(`Processing custom file: ${customFile.filename}, version: ${customFile.version || 'undefined'}`);
+    
+    // Combine global ignores with custom ignores
+    const combinedIgnores = [...ignoreFiles];
+    if (customFile.ignorePatterns) {
+      combinedIgnores.push(...customFile.ignorePatterns);
+    }
+    
+    // Process files according to the custom configuration
+    const customDocs = await processFilesWithPatterns(
+      context,
+      allDocFiles,
+      customFile.includePatterns,
+      combinedIgnores,
+      customFile.orderPatterns || [],
+      customFile.includeUnmatchedLast ?? false
+    );
+    
+    if (customDocs.length > 0) {
+      // Use custom title/description or fall back to defaults
+      const customTitle = customFile.title || docTitle;
+      const customDescription = customFile.description || docDescription;
+      
+      // Generate the custom LLM file
+      const customFilePath = path.join(outDir, customFile.filename);
+      await generateLLMFile(
+        customDocs,
+        customFilePath,
+        customTitle,
+        customDescription,
+        customFile.fullContent,
+        customFile.version
+      );
+      
+      console.log(`Generated custom LLM file: ${customFile.filename} with ${customDocs.length} documents`);
+    } else {
+      console.warn(`No matching documents found for custom LLM file: ${customFile.filename}`);
+    }
+  }
+}
+
+/**
+ * Collect all document files from docs directory and optionally blog
+ * @param context - Plugin context
+ * @returns Array of file paths
+ */
+export async function collectDocFiles(context: PluginContext): Promise<string[]> {
+  const { siteDir, docsDir, options } = context;
+  const { ignoreFiles = [], includeBlog = false } = options;
+  
+  const allDocFiles: string[] = [];
+  
+  // Process docs directory
+  const fullDocsDir = path.join(siteDir, docsDir);
+  
+  try {
+    await fs.access(fullDocsDir);
+    
+    // Collect all markdown files from docs directory
+    const docFiles = await readMarkdownFiles(fullDocsDir, siteDir, ignoreFiles);
+    allDocFiles.push(...docFiles);
+    
+  } catch (err) {
+    console.warn(`Docs directory not found: ${fullDocsDir}`);
+  }
+  
+  // Process blog if enabled
+  if (includeBlog) {
+    const blogDir = path.join(siteDir, 'blog');
+    
+    try {
+      await fs.access(blogDir);
+      
+      // Collect all markdown files from blog directory
+      const blogFiles = await readMarkdownFiles(blogDir, siteDir, ignoreFiles);
+      allDocFiles.push(...blogFiles);
+      
+    } catch (err) {
+      console.warn(`Blog directory not found: ${blogDir}`);
+    }
+  }
+  
+  return allDocFiles;
+}