docusaurus-plugin-llms 0.1.1 → 0.1.3

package/lib/index.js

@@ -8,184 +8,9 @@
  *
  * The plugin runs during the Docusaurus build process and scans all Markdown files in the docs directory.
  */
-var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    var desc = Object.getOwnPropertyDescriptor(m, k);
-    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
-      desc = { enumerable: true, get: function() { return m[k]; } };
-    }
-    Object.defineProperty(o, k2, desc);
-}) : (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    o[k2] = m[k];
-}));
-var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
-    Object.defineProperty(o, "default", { enumerable: true, value: v });
-}) : function(o, v) {
-    o["default"] = v;
-});
-var __importStar = (this && this.__importStar) || (function () {
-    var ownKeys = function(o) {
-        ownKeys = Object.getOwnPropertyNames || function (o) {
-            var ar = [];
-            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
-            return ar;
-        };
-        return ownKeys(o);
-    };
-    return function (mod) {
-        if (mod && mod.__esModule) return mod;
-        var result = {};
-        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
-        __setModuleDefault(result, mod);
-        return result;
-    };
-})();
-var __importDefault = (this && this.__importDefault) || function (mod) {
-    return (mod && mod.__esModule) ? mod : { "default": mod };
-};
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.default = docusaurusPluginLLMs;
-const fs = __importStar(require("fs/promises"));
-const path = __importStar(require("path"));
-const gray_matter_1 = __importDefault(require("gray-matter"));
-const minimatch_1 = require("minimatch");
-/**
- * Write content to a file
- * @param filePath - Path to write the file to
- * @param data - Content to write
- */
-async function writeFile(filePath, data) {
-    return fs.writeFile(filePath, data, 'utf8');
-}
-/**
- * Read content from a file
- * @param filePath - Path of the file to read
- * @returns Content of the file
- */
-async function readFile(filePath) {
-    return fs.readFile(filePath, 'utf8');
-}
-/**
- * Check if a file should be ignored based on glob patterns
- * @param filePath - Path to the file
- * @param baseDir - Base directory for relative paths
- * @param ignorePatterns - Glob patterns for files to ignore
- * @returns Whether the file should be ignored
- */
-function shouldIgnoreFile(filePath, baseDir, ignorePatterns) {
-    if (ignorePatterns.length === 0) {
-        return false;
-    }
-    const relativePath = path.relative(baseDir, filePath);
-    return ignorePatterns.some(pattern => (0, minimatch_1.minimatch)(relativePath, pattern, { matchBase: true }));
-}
-/**
- * Recursively reads all Markdown files in a directory
- * @param dir - Directory to scan
- * @param baseDir - Base directory for relative paths
- * @param ignorePatterns - Glob patterns for files to ignore
- * @returns Array of file paths
- */
-async function readMarkdownFiles(dir, baseDir, ignorePatterns = []) {
-    const files = [];
-    const entries = await fs.readdir(dir, { withFileTypes: true });
-    for (const entry of entries) {
-        const fullPath = path.join(dir, entry.name);
-        if (shouldIgnoreFile(fullPath, baseDir, ignorePatterns)) {
-            continue;
-        }
-        if (entry.isDirectory()) {
-            const subDirFiles = await readMarkdownFiles(fullPath, baseDir, ignorePatterns);
-            files.push(...subDirFiles);
-        }
-        else if (entry.name.endsWith('.md') || entry.name.endsWith('.mdx')) {
-            files.push(fullPath);
-        }
-    }
-    return files;
-}
-/**
- * Extract title from content or use the filename
- * @param data - Frontmatter data
- * @param content - Markdown content
- * @param filePath - Path to the file
- * @returns Extracted title
- */
-function extractTitle(data, content, filePath) {
-    // First try frontmatter
-    if (data.title) {
-        return data.title;
-    }
-    // Then try first heading
-    const headingMatch = content.match(/^#\s+(.*)/m);
-    if (headingMatch) {
-        return headingMatch[1].trim();
-    }
-    // Finally use filename
-    return path.basename(filePath, path.extname(filePath))
-        .replace(/-/g, ' ')
-        .replace(/\b\w/g, c => c.toUpperCase());
-}
-/**
- * Clean markdown content for LLM consumption
- * @param content - Raw markdown content
- * @returns Cleaned content
- */
-function cleanMarkdownContent(content) {
-    // Remove HTML tags
-    let cleaned = content.replace(/<[^>]*>/g, '');
-    // Normalize whitespace
-    cleaned = cleaned.replace(/\r\n/g, '\n')
-        .replace(/\n{3,}/g, '\n\n')
-        .trim();
-    return cleaned;
-}
-/**
- * Process a markdown file and extract its metadata and content
- * @param filePath - Path to the markdown file
- * @param baseDir - Base directory
- * @param siteUrl - Base URL of the site
- * @param pathPrefix - Path prefix for URLs (e.g., 'docs' or 'blog')
- * @returns Processed file data
- */
-async function processMarkdownFile(filePath, baseDir, siteUrl, pathPrefix = 'docs') {
-    const content = await readFile(filePath);
-    const { data, content: markdownContent } = (0, gray_matter_1.default)(content);
-    const relativePath = path.relative(baseDir, filePath);
-    // Convert to URL path format (replace backslashes with forward slashes on Windows)
-    const normalizedPath = relativePath.replace(/\\/g, '/');
-    // Convert .md extension to appropriate path
-    const linkPathBase = normalizedPath.replace(/\.mdx?$/, '');
-    // Handle index files specially
-    const linkPath = linkPathBase.endsWith('index')
-        ? linkPathBase.replace(/\/index$/, '')
-        : linkPathBase;
-    // Generate full URL
-    const fullUrl = new URL(`${pathPrefix}/${linkPath}`, siteUrl).toString();
-    // Extract title
-    const title = extractTitle(data, markdownContent, filePath);
-    // Get description from frontmatter or first paragraph
-    let description = data.description || '';
-    if (!description) {
-        const paragraphs = markdownContent.split('\n\n');
-        for (const para of paragraphs) {
-            if (para.trim() && !para.startsWith('#')) {
-                description = para.trim();
-                break;
-            }
-        }
-    }
-    // Clean and process content
-    const cleanedContent = cleanMarkdownContent(markdownContent);
-    return {
-        title,
-        path: normalizedPath,
-        url: fullUrl,
-        content: cleanedContent,
-        description: description || '',
-    };
-}
+const generator_1 = require("./generator");
 /**
  * A Docusaurus plugin to generate LLM-friendly documentation following
  * the llmtxt.org standard
@@ -196,8 +21,36 @@ async function processMarkdownFile(filePath, baseDir, siteUrl, pathPrefix = 'doc
  */
 function docusaurusPluginLLMs(context, options = {}) {
     // Set default options
-    const { generateLLMsTxt = true, generateLLMsFullTxt = true, docsDir = 'docs', ignoreFiles = [], title, description, llmsTxtFilename = 'llms.txt', llmsFullTxtFilename = 'llms-full.txt', includeBlog = false, } = options;
+    const { generateLLMsTxt = true, generateLLMsFullTxt = true, docsDir = 'docs', ignoreFiles = [], title, description, llmsTxtFilename = 'llms.txt', llmsFullTxtFilename = 'llms-full.txt', includeBlog = false, pathTransformation, includeOrder = [], includeUnmatchedLast = true, customLLMFiles = [], } = options;
     const { siteDir, siteConfig, outDir, } = context;
+    // Build the site URL with proper trailing slash
+    const siteUrl = siteConfig.url + (siteConfig.baseUrl.endsWith('/')
+        ? siteConfig.baseUrl.slice(0, -1)
+        : siteConfig.baseUrl || '');
+    // Create a plugin context object with processed options
+    const pluginContext = {
+        siteDir,
+        outDir,
+        siteUrl,
+        docsDir,
+        docTitle: title || siteConfig.title,
+        docDescription: description || siteConfig.tagline || '',
+        options: {
+            generateLLMsTxt,
+            generateLLMsFullTxt,
+            docsDir,
+            ignoreFiles,
+            title,
+            description,
+            llmsTxtFilename,
+            llmsFullTxtFilename,
+            includeBlog,
+            pathTransformation,
+            includeOrder,
+            includeUnmatchedLast,
+            customLLMFiles,
+        }
+    };
     return {
         name: 'docusaurus-plugin-llms',
         /**
@@ -205,122 +58,20 @@ function docusaurusPluginLLMs(context, options = {}) {
          */
         async postBuild() {
             console.log('Generating LLM-friendly documentation...');
-            // Custom title and description or fallback to site values
-            const docTitle = title || siteConfig.title;
-            const docDescription = description || siteConfig.tagline || '';
-            // Build the site URL with proper trailing slash
-            const siteUrl = siteConfig.url + (siteConfig.baseUrl.endsWith('/')
-                ? siteConfig.baseUrl.slice(0, -1)
-                : siteConfig.baseUrl || '');
-            // Initialize docs collection
-            const allDocs = [];
             try {
-                // Process docs directory
-                const fullDocsDir = path.join(siteDir, docsDir);
-                try {
-                    await fs.access(fullDocsDir);
-                    // Collect all markdown files from docs directory
-                    const docFiles = await readMarkdownFiles(fullDocsDir, siteDir, ignoreFiles);
-                    if (docFiles.length > 0) {
-                        // Process each file
-                        for (const filePath of docFiles) {
-                            try {
-                                const docInfo = await processMarkdownFile(filePath, fullDocsDir, siteUrl, 'docs');
-                                allDocs.push(docInfo);
-                            }
-                            catch (err) {
-                                console.warn(`Error processing ${filePath}: ${err.message}`);
-                            }
-                        }
-                        console.log(`Processed ${docFiles.length} documentation files`);
-                    }
-                    else {
-                        console.warn('No markdown files found in docs directory.');
-                    }
-                }
-                catch (err) {
-                    console.warn(`Docs directory not found: ${fullDocsDir}`);
-                }
-                // Process blog if enabled
-                if (includeBlog) {
-                    const blogDir = path.join(siteDir, 'blog');
-                    try {
-                        await fs.access(blogDir);
-                        // Collect all markdown files from blog directory
-                        const blogFiles = await readMarkdownFiles(blogDir, siteDir, ignoreFiles);
-                        if (blogFiles.length > 0) {
-                            // Process each file
-                            for (const filePath of blogFiles) {
-                                try {
-                                    const docInfo = await processMarkdownFile(filePath, blogDir, siteUrl, 'blog');
-                                    allDocs.push(docInfo);
-                                }
-                                catch (err) {
-                                    console.warn(`Error processing ${filePath}: ${err.message}`);
-                                }
-                            }
-                            console.log(`Processed ${blogFiles.length} blog files`);
-                        }
-                        else {
-                            console.warn('No markdown files found in blog directory.');
-                        }
-                    }
-                    catch (err) {
-                        console.warn(`Blog directory not found: ${blogDir}`);
-                    }
-                }
+                // Collect all document files
+                const allDocFiles = await (0, generator_1.collectDocFiles)(pluginContext);
                 // Skip further processing if no documents were found
-                if (allDocs.length === 0) {
+                if (allDocFiles.length === 0) {
                     console.warn('No documents found to process.');
                     return;
                 }
-                // Sort files to ensure consistent ordering
-                allDocs.sort((a, b) => a.title.localeCompare(b.title));
-                // Generate llms.txt
-                if (generateLLMsTxt) {
-                    const llmsTxtPath = path.join(outDir, llmsTxtFilename);
-                    const tocItems = allDocs.map(doc => {
-                        return `- [${doc.title}](${doc.url})${doc.description ? `: ${doc.description.split('\n')[0]}` : ''}`;
-                    });
-                    const llmsTxtContent = `# ${docTitle}
-
-> ${docDescription}
-
-This file contains links to all documentation sections following the llmtxt.org standard.
-
-## Table of Contents
-
-${tocItems.join('\n')}
-`;
-                    await writeFile(llmsTxtPath, llmsTxtContent);
-                    console.log(`Generated ${llmsTxtFilename}: ${llmsTxtPath}`);
-                }
-                // Generate llms-full.txt with all content
-                if (generateLLMsFullTxt) {
-                    const llmsFullTxtPath = path.join(outDir, llmsFullTxtFilename);
-                    const fullContentSections = allDocs.map(doc => {
-                        return `## ${doc.title}
-
-${doc.content}`;
-                    });
-                    const llmsFullTxtContent = `# ${docTitle}
-
-> ${docDescription}
-
-This file contains all documentation content in a single document following the llmtxt.org standard.
-
-${fullContentSections.join('\n\n---\n\n')}
-`;
-                    await writeFile(llmsFullTxtPath, llmsFullTxtContent);
-                    console.log(`Generated ${llmsFullTxtFilename}: ${llmsFullTxtPath}`);
-                }
-                // Output statistics
-                const stats = {
-                    totalDocuments: allDocs.length,
-                    totalBytes: allDocs.reduce((sum, doc) => sum + doc.content.length, 0),
-                    approxTokens: Math.round(allDocs.reduce((sum, doc) => sum + doc.content.length, 0) / 4), // Rough token estimate
-                };
-                console.log(`Stats: ${stats.totalDocuments} documents, ${Math.round(stats.totalBytes / 1024)}KB, ~${stats.approxTokens} tokens`);
+                // Process standard LLM files (llms.txt and llms-full.txt)
+                await (0, generator_1.generateStandardLLMFiles)(pluginContext, allDocFiles);
+                // Process custom LLM files
+                await (0, generator_1.generateCustomLLMFiles)(pluginContext, allDocFiles);
+                // Output overall statistics
+                console.log(`Stats: ${allDocFiles.length} total available documents processed`);
             }
             catch (err) {
                 console.error('Error generating LLM documentation:', err);

package/lib/processor.d.ts

@@ -0,0 +1,28 @@
+/**
+ * Document processing functions for the docusaurus-plugin-llms plugin
+ */
+import { DocInfo, PluginContext } from './types';
+/**
+ * Process a markdown file and extract its metadata and content
+ * @param filePath - Path to the markdown file
+ * @param baseDir - Base directory
+ * @param siteUrl - Base URL of the site
+ * @param pathPrefix - Path prefix for URLs (e.g., 'docs' or 'blog')
+ * @param pathTransformation - Path transformation configuration
+ * @returns Processed file data
+ */
+export declare function processMarkdownFile(filePath: string, baseDir: string, siteUrl: string, pathPrefix?: string, pathTransformation?: {
+    ignorePaths?: string[];
+    addPaths?: string[];
+}): Promise<DocInfo>;
+/**
+ * Process files based on include patterns, ignore patterns, and ordering
+ * @param context - Plugin context
+ * @param allFiles - All available files
+ * @param includePatterns - Patterns for files to include
+ * @param ignorePatterns - Patterns for files to ignore
+ * @param orderPatterns - Patterns for ordering files
+ * @param includeUnmatched - Whether to include unmatched files
+ * @returns Processed files
+ */
+export declare function processFilesWithPatterns(context: PluginContext, allFiles: string[], includePatterns?: string[], ignorePatterns?: string[], orderPatterns?: string[], includeUnmatched?: boolean): Promise<DocInfo[]>;

package/lib/processor.js

@@ -0,0 +1,211 @@
+"use strict";
+/**
+ * Document processing functions for the docusaurus-plugin-llms plugin
+ */
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.processMarkdownFile = processMarkdownFile;
+exports.processFilesWithPatterns = processFilesWithPatterns;
+const path = __importStar(require("path"));
+const gray_matter_1 = __importDefault(require("gray-matter"));
+const minimatch_1 = require("minimatch");
+const utils_1 = require("./utils");
+/**
+ * Process a markdown file and extract its metadata and content
+ * @param filePath - Path to the markdown file
+ * @param baseDir - Base directory
+ * @param siteUrl - Base URL of the site
+ * @param pathPrefix - Path prefix for URLs (e.g., 'docs' or 'blog')
+ * @param pathTransformation - Path transformation configuration
+ * @returns Processed file data
+ */
+async function processMarkdownFile(filePath, baseDir, siteUrl, pathPrefix = 'docs', pathTransformation) {
+    const content = await (0, utils_1.readFile)(filePath);
+    const { data, content: markdownContent } = (0, gray_matter_1.default)(content);
+    const relativePath = path.relative(baseDir, filePath);
+    // Convert to URL path format (replace backslashes with forward slashes on Windows)
+    const normalizedPath = relativePath.replace(/\\/g, '/');
+    // Convert .md extension to appropriate path
+    const linkPathBase = normalizedPath.replace(/\.mdx?$/, '');
+    // Handle index files specially
+    const linkPath = linkPathBase.endsWith('index')
+        ? linkPathBase.replace(/\/index$/, '')
+        : linkPathBase;
+    // Apply path transformations to the link path
+    const transformedLinkPath = (0, utils_1.applyPathTransformations)(linkPath, pathTransformation);
+    // Also apply path transformations to the pathPrefix if it's not empty
+    // This allows removing 'docs' from the path when specified in ignorePaths
+    let transformedPathPrefix = pathPrefix;
+    if (pathPrefix && pathTransformation?.ignorePaths?.includes(pathPrefix)) {
+        transformedPathPrefix = '';
+    }
+    // Generate full URL with transformed path and path prefix
+    const fullUrl = new URL(`${transformedPathPrefix ? `${transformedPathPrefix}/` : ''}${transformedLinkPath}`, siteUrl).toString();
+    // Extract title
+    const title = (0, utils_1.extractTitle)(data, markdownContent, filePath);
+    // Get description from frontmatter or first paragraph
+    let description = '';
+    // First priority: Use frontmatter description if available
+    if (data.description) {
+        description = data.description;
+    }
+    else {
+        // Second priority: Find the first non-heading paragraph
+        const paragraphs = markdownContent.split('\n\n');
+        for (const para of paragraphs) {
+            const trimmedPara = para.trim();
+            // Skip empty paragraphs and headings
+            if (trimmedPara && !trimmedPara.startsWith('#')) {
+                description = trimmedPara;
+                break;
+            }
+        }
+        // Third priority: If still no description, use the first heading's content
+        if (!description) {
+            const firstHeadingMatch = markdownContent.match(/^#\s+(.*?)$/m);
+            if (firstHeadingMatch && firstHeadingMatch[1]) {
+                description = firstHeadingMatch[1].trim();
+            }
+        }
+    }
+    // Only remove heading markers at the beginning of descriptions or lines
+    // This preserves # characters that are part of the content
+    if (description) {
+        // Original approach had issues with hashtags inside content
+        // Fix: Only remove # symbols at the beginning of lines or description
+        // that are followed by a space (actual heading markers)
+        description = description.replace(/^(#+)\s+/gm, '');
+        // Special handling for description frontmatter with heading markers
+        if (data.description && data.description.startsWith('#')) {
+            // If the description in frontmatter starts with a heading marker,
+            // we should preserve it in the extracted description
+            description = description.replace(/^#+\s+/, '');
+        }
+        // Preserve inline hashtags (not heading markers)
+        // We don't want to treat hashtags in the middle of content as headings
+        // Validate that the description doesn't contain markdown headings
+        if (description.match(/^#+\s+/m)) {
+            console.warn(`Warning: Description for "${title}" may still contain heading markers`);
+        }
+        // Warn if the description contains HTML tags
+        if (/<[^>]+>/g.test(description)) {
+            console.warn(`Warning: Description for "${title}" contains HTML tags`);
+        }
+        // Warn if the description is very long
+        if (description.length > 500) {
+            console.warn(`Warning: Description for "${title}" is very long (${description.length} characters)`);
+        }
+    }
+    // Clean and process content
+    const cleanedContent = (0, utils_1.cleanMarkdownContent)(markdownContent);
+    return {
+        title,
+        path: normalizedPath,
+        url: fullUrl,
+        content: cleanedContent,
+        description: description || '',
+    };
+}
+/**
+ * Process files based on include patterns, ignore patterns, and ordering
+ * @param context - Plugin context
+ * @param allFiles - All available files
+ * @param includePatterns - Patterns for files to include
+ * @param ignorePatterns - Patterns for files to ignore
+ * @param orderPatterns - Patterns for ordering files
+ * @param includeUnmatched - Whether to include unmatched files
+ * @returns Processed files
+ */
+async function processFilesWithPatterns(context, allFiles, includePatterns = [], ignorePatterns = [], orderPatterns = [], includeUnmatched = false) {
+    const { siteDir, siteUrl, docsDir } = context;
+    // Filter files based on include patterns
+    let filteredFiles = allFiles;
+    if (includePatterns.length > 0) {
+        filteredFiles = allFiles.filter(file => {
+            const relativePath = path.relative(siteDir, file);
+            return includePatterns.some(pattern => (0, minimatch_1.minimatch)(relativePath, pattern, { matchBase: true }));
+        });
+    }
+    // Apply ignore patterns
+    if (ignorePatterns.length > 0) {
+        filteredFiles = filteredFiles.filter(file => {
+            const relativePath = path.relative(siteDir, file);
+            return !ignorePatterns.some(pattern => (0, minimatch_1.minimatch)(relativePath, pattern, { matchBase: true }));
+        });
+    }
+    // Order files according to orderPatterns
+    let filesToProcess = [];
+    if (orderPatterns.length > 0) {
+        const matchedFiles = new Set();
+        // Process files according to orderPatterns
+        for (const pattern of orderPatterns) {
+            const matchingFiles = filteredFiles.filter(file => {
+                const relativePath = path.relative(siteDir, file);
+                return (0, minimatch_1.minimatch)(relativePath, pattern, { matchBase: true }) && !matchedFiles.has(file);
+            });
+            for (const file of matchingFiles) {
+                filesToProcess.push(file);
+                matchedFiles.add(file);
+            }
+        }
+        // Add remaining files if includeUnmatched is true
+        if (includeUnmatched) {
+            const remainingFiles = filteredFiles.filter(file => !matchedFiles.has(file));
+            filesToProcess.push(...remainingFiles);
+        }
+    }
+    else {
+        filesToProcess = filteredFiles;
+    }
+    // Process each file to generate DocInfo
+    const processedDocs = [];
+    for (const filePath of filesToProcess) {
+        try {
+            // Determine if this is a blog or docs file
+            const isBlogFile = filePath.includes(path.join(siteDir, 'blog'));
+            const baseDir = isBlogFile ? path.join(siteDir, 'blog') : path.join(siteDir, docsDir);
+            const pathPrefix = isBlogFile ? 'blog' : 'docs';
+            const docInfo = await processMarkdownFile(filePath, baseDir, siteUrl, pathPrefix, context.options.pathTransformation);
+            processedDocs.push(docInfo);
+        }
+        catch (err) {
+            console.warn(`Error processing ${filePath}: ${err.message}`);
+        }
+    }
+    return processedDocs;
+}

package/lib/utils.d.ts

@@ -0,0 +1,53 @@
+/**
+ * Utility functions for the docusaurus-plugin-llms plugin
+ */
+import { PluginOptions } from './types';
+/**
+ * Write content to a file
+ * @param filePath - Path to write the file to
+ * @param data - Content to write
+ */
+export declare function writeFile(filePath: string, data: string): Promise<void>;
+/**
+ * Read content from a file
+ * @param filePath - Path of the file to read
+ * @returns Content of the file
+ */
+export declare function readFile(filePath: string): Promise<string>;
+/**
+ * Check if a file should be ignored based on glob patterns
+ * @param filePath - Path to the file
+ * @param baseDir - Base directory for relative paths
+ * @param ignorePatterns - Glob patterns for files to ignore
+ * @returns Whether the file should be ignored
+ */
+export declare function shouldIgnoreFile(filePath: string, baseDir: string, ignorePatterns: string[]): boolean;
+/**
+ * Recursively reads all Markdown files in a directory
+ * @param dir - Directory to scan
+ * @param baseDir - Base directory for relative paths
+ * @param ignorePatterns - Glob patterns for files to ignore
+ * @returns Array of file paths
+ */
+export declare function readMarkdownFiles(dir: string, baseDir: string, ignorePatterns?: string[]): Promise<string[]>;
+/**
+ * Extract title from content or use the filename
+ * @param data - Frontmatter data
+ * @param content - Markdown content
+ * @param filePath - Path to the file
+ * @returns Extracted title
+ */
+export declare function extractTitle(data: any, content: string, filePath: string): string;
+/**
+ * Clean markdown content for LLM consumption
+ * @param content - Raw markdown content
+ * @returns Cleaned content
+ */
+export declare function cleanMarkdownContent(content: string): string;
+/**
+ * Apply path transformations according to configuration
+ * @param urlPath - Original URL path
+ * @param pathTransformation - Path transformation configuration
+ * @returns Transformed URL path
+ */
+export declare function applyPathTransformations(urlPath: string, pathTransformation?: PluginOptions['pathTransformation']): string;