docusaurus-plugin-llms 0.2.2 → 0.3.0

package/lib/processor.js

@@ -61,15 +61,39 @@ async function processMarkdownFile(filePath, baseDir, siteUrl, pathPrefix = 'doc
     if (data.draft === true) {
         return null;
     }
+    // Validate and clean empty frontmatter fields
+    // Empty strings should be treated as undefined to allow fallback logic
+    if (data.title !== undefined && !(0, utils_1.isNonEmptyString)(data.title)) {
+        utils_1.logger.warn(`Empty title in frontmatter for ${filePath}. Using fallback.`);
+        data.title = undefined;
+    }
+    if (data.description !== undefined && !(0, utils_1.isNonEmptyString)(data.description)) {
+        data.description = undefined;
+    }
+    if (data.slug !== undefined && !(0, utils_1.isNonEmptyString)(data.slug)) {
+        data.slug = undefined;
+    }
+    if (data.id !== undefined && !(0, utils_1.isNonEmptyString)(data.id)) {
+        data.id = undefined;
+    }
     // Resolve partial imports before processing
     const resolvedContent = await (0, utils_1.resolvePartialImports)(markdownContent, filePath);
     const relativePath = path.relative(baseDir, filePath);
     // Convert to URL path format (replace backslashes with forward slashes on Windows)
-    const normalizedPath = relativePath.replace(/\\/g, '/');
+    const normalizedPath = (0, utils_1.normalizePath)(relativePath);
     let fullUrl;
-    if (resolvedUrl) {
+    if ((0, utils_1.isNonEmptyString)(resolvedUrl)) {
         // Use the actual resolved URL from Docusaurus if provided
-        fullUrl = new URL(resolvedUrl, siteUrl).toString();
+        try {
+            fullUrl = new URL(resolvedUrl, siteUrl).toString();
+        }
+        catch (error) {
+            utils_1.logger.warn(`Invalid URL construction: ${resolvedUrl} with base ${siteUrl}. Using fallback.`);
+            // Fallback to string concatenation with proper path joining
+            const baseUrl = siteUrl.endsWith('/') ? siteUrl.slice(0, -1) : siteUrl;
+            const urlPath = resolvedUrl.startsWith('/') ? resolvedUrl : `/${resolvedUrl}`;
+            fullUrl = baseUrl + urlPath;
+        }
     }
     else {
         // Fallback to the old path construction method
@@ -95,15 +119,49 @@ async function processMarkdownFile(filePath, baseDir, siteUrl, pathPrefix = 'doc
         if (pathPrefix && pathTransformation?.ignorePaths?.includes(pathPrefix)) {
             transformedPathPrefix = '';
         }
-        // Generate full URL with transformed path and path prefix
-        fullUrl = new URL(`${transformedPathPrefix ? `${transformedPathPrefix}/` : ''}${transformedLinkPath}`, siteUrl).toString();
+        // Ensure path segments are URL-safe with sophisticated encoding detection
+        const encodedLinkPath = transformedLinkPath.split('/').map(segment => {
+            // Check if segment contains characters that need encoding
+            // Unreserved characters (per RFC 3986): A-Z a-z 0-9 - . _ ~
+            if (!/[^A-Za-z0-9\-._~]/.test(segment)) {
+                // Segment only contains unreserved characters, no encoding needed
+                return segment;
+            }
+            try {
+                // Try to decode - if it changes, it was already encoded
+                const decoded = decodeURIComponent(segment);
+                if (decoded !== segment) {
+                    // Was already encoded, return as-is
+                    return segment;
+                }
+                // Not encoded, encode it
+                return encodeURIComponent(segment);
+            }
+            catch {
+                // Malformed encoding, re-encode
+                return encodeURIComponent(segment);
+            }
+        }).join('/');
+        // Construct URL by encoding path components, then combine with site URL
+        // We don't use URL constructor for the full path because it decodes some characters
+        const pathPart = transformedPathPrefix ? `${transformedPathPrefix}/${encodedLinkPath}` : encodedLinkPath;
+        try {
+            const baseUrl = new URL(siteUrl);
+            fullUrl = `${baseUrl.origin}/${pathPart}`;
+        }
+        catch (error) {
+            utils_1.logger.warn(`Invalid siteUrl: ${siteUrl}. Using fallback.`);
+            // Fallback to string concatenation with proper path joining
+            const baseUrl = siteUrl.endsWith('/') ? siteUrl.slice(0, -1) : siteUrl;
+            fullUrl = `${baseUrl}/${pathPart}`;
+        }
     }
     // Extract title
     const title = (0, utils_1.extractTitle)(data, resolvedContent, filePath);
     // Get description from frontmatter or first paragraph
     let description = '';
     // First priority: Use frontmatter description if available
-    if (data.description) {
+    if ((0, utils_1.isNonEmptyString)(data.description)) {
         description = data.description;
     }
     else {
@@ -127,13 +185,13 @@ async function processMarkdownFile(filePath, baseDir, siteUrl, pathPrefix = 'doc
     }
     // Only remove heading markers at the beginning of descriptions or lines
     // This preserves # characters that are part of the content
-    if (description) {
+    if ((0, utils_1.isNonEmptyString)(description)) {
         // Original approach had issues with hashtags inside content
         // Fix: Only remove # symbols at the beginning of lines or description
         // that are followed by a space (actual heading markers)
         description = description.replace(/^(#+)\s+/gm, '');
         // Special handling for description frontmatter with heading markers
-        if (data.description && data.description.startsWith('#')) {
+        if ((0, utils_1.isNonEmptyString)(data.description) && data.description.startsWith('#')) {
             // If the description in frontmatter starts with a heading marker,
             // we should preserve it in the extracted description
             description = description.replace(/^#+\s+/, '');
@@ -142,15 +200,15 @@ async function processMarkdownFile(filePath, baseDir, siteUrl, pathPrefix = 'doc
         // We don't want to treat hashtags in the middle of content as headings
         // Validate that the description doesn't contain markdown headings
         if (description.match(/^#+\s+/m)) {
-            console.warn(`Warning: Description for "${title}" may still contain heading markers`);
+            utils_1.logger.warn(`Warning: Description for "${title}" may still contain heading markers`);
         }
         // Warn if the description contains HTML tags
         if (/<[^>]+>/g.test(description)) {
-            console.warn(`Warning: Description for "${title}" contains HTML tags`);
+            utils_1.logger.warn(`Warning: Description for "${title}" contains HTML tags`);
         }
         // Warn if the description is very long
         if (description.length > 500) {
-            console.warn(`Warning: Description for "${title}" is very long (${description.length} characters)`);
+            utils_1.logger.warn(`Warning: Description for "${title}" is very long (${description.length} characters)`);
         }
     }
     // Clean and process content (now with partials already resolved)
@@ -164,6 +222,110 @@ async function processMarkdownFile(filePath, baseDir, siteUrl, pathPrefix = 'doc
         frontMatter: data,
     };
 }
+/**
+ * Remove numbered prefixes from path segments (e.g., "01-intro" -> "intro")
+ */
+function removeNumberedPrefixes(path) {
+    return path.split('/').map(segment => {
+        // Remove numbered prefixes like "01-", "1-", "001-" from each segment
+        return segment.replace(/^\d+-/, '');
+    }).join('/');
+}
+/**
+ * Try to find a route in the route map from a list of possible paths
+ */
+function findRouteInMap(routeMap, possiblePaths) {
+    for (const possiblePath of possiblePaths) {
+        const route = routeMap.get(possiblePath) || routeMap.get(possiblePath + '/');
+        if (route) {
+            return route;
+        }
+    }
+    return undefined;
+}
+/**
+ * Try exact match for route resolution
+ */
+function tryExactRouteMatch(routeMap, relativePath, pathPrefix) {
+    const possiblePaths = [
+        `/${pathPrefix}/${relativePath}`,
+        `/${relativePath}`,
+    ];
+    return findRouteInMap(routeMap, possiblePaths);
+}
+/**
+ * Try route resolution with numbered prefix removal
+ */
+function tryNumberedPrefixResolution(routeMap, relativePath, pathPrefix) {
+    const cleanPath = removeNumberedPrefixes(relativePath);
+    // Try basic cleaned path
+    const basicPaths = [`/${pathPrefix}/${cleanPath}`, `/${cleanPath}`];
+    const basicMatch = findRouteInMap(routeMap, basicPaths);
+    if (basicMatch) {
+        return basicMatch;
+    }
+    // Try nested folder structures with numbered prefixes at different levels
+    const segments = relativePath.split('/');
+    if (segments.length > 1) {
+        for (let i = 0; i < segments.length; i++) {
+            const modifiedSegments = [...segments];
+            modifiedSegments[i] = modifiedSegments[i].replace(/^\d+-/, '');
+            const modifiedPath = modifiedSegments.join('/');
+            const pathsToTry = [`/${pathPrefix}/${modifiedPath}`, `/${modifiedPath}`];
+            const match = findRouteInMap(routeMap, pathsToTry);
+            if (match) {
+                return match;
+            }
+        }
+    }
+    return undefined;
+}
+/**
+ * Try finding best match using routes paths array
+ */
+function tryRoutesPathsMatch(routesPaths, relativePath, pathPrefix) {
+    const cleanPath = removeNumberedPrefixes(relativePath);
+    const normalizedCleanPath = cleanPath.toLowerCase();
+    return routesPaths.find(routePath => {
+        const normalizedRoute = routePath.toLowerCase();
+        return normalizedRoute.endsWith(`/${normalizedCleanPath}`) ||
+            normalizedRoute === `/${pathPrefix}/${normalizedCleanPath}` ||
+            normalizedRoute === `/${normalizedCleanPath}`;
+    });
+}
+/**
+ * Resolve the URL for a document using Docusaurus routes
+ * @param filePath - Full path to the file
+ * @param baseDir - Base directory (typically siteDir)
+ * @param pathPrefix - Path prefix ('docs' or 'blog')
+ * @param context - Plugin context with route map
+ * @returns Resolved URL or undefined if not found
+ */
+function resolveDocumentUrl(filePath, baseDir, pathPrefix, context) {
+    // Early return if no route map available
+    if (!context.routeMap) {
+        return undefined;
+    }
+    // Convert file path to a potential route path
+    const relativePath = (0, utils_1.normalizePath)(path.relative(baseDir, filePath))
+        .replace(/\.mdx?$/, '')
+        .replace(/\/index$/, '');
+    // Try exact match first (respects Docusaurus's resolved routes)
+    const exactMatch = tryExactRouteMatch(context.routeMap, relativePath, pathPrefix);
+    if (exactMatch) {
+        return exactMatch;
+    }
+    // Try numbered prefix removal as fallback
+    const prefixMatch = tryNumberedPrefixResolution(context.routeMap, relativePath, pathPrefix);
+    if (prefixMatch) {
+        return prefixMatch;
+    }
+    // Try to find the best match using the routesPaths array
+    if (context.routesPaths) {
+        return tryRoutesPathsMatch(context.routesPaths, relativePath, pathPrefix);
+    }
+    return undefined;
+}
 /**
  * Process files based on include patterns, ignore patterns, and ordering
  * @param context - Plugin context
@@ -174,21 +336,44 @@ async function processMarkdownFile(filePath, baseDir, siteUrl, pathPrefix = 'doc
  * @param includeUnmatched - Whether to include unmatched files
  * @returns Processed files
  */
+/**
+ * Helper function to check if a file matches a pattern
+ * Tries matching against multiple path variants for better usability
+ */
+function matchesPattern(file, pattern, siteDir, docsDir) {
+    const minimatchOptions = { matchBase: true };
+    // Get site-relative path (e.g., "docs/quickstart/file.md")
+    const siteRelativePath = (0, utils_1.normalizePath)(path.relative(siteDir, file));
+    // Get docs-relative path (e.g., "quickstart/file.md")
+    // Normalize both paths to handle different path separators and resolve any .. or .
+    const docsBaseDir = path.resolve(path.join(siteDir, docsDir));
+    const resolvedFile = path.resolve(file);
+    const docsRelativePath = resolvedFile.startsWith(docsBaseDir)
+        ? (0, utils_1.normalizePath)(path.relative(docsBaseDir, resolvedFile))
+        : null;
+    // Try matching against site-relative path
+    if ((0, minimatch_1.minimatch)(siteRelativePath, pattern, minimatchOptions)) {
+        return true;
+    }
+    // Try matching against docs-relative path if available
+    if (docsRelativePath && (0, minimatch_1.minimatch)(docsRelativePath, pattern, minimatchOptions)) {
+        return true;
+    }
+    return false;
+}
 async function processFilesWithPatterns(context, allFiles, includePatterns = [], ignorePatterns = [], orderPatterns = [], includeUnmatched = false) {
     const { siteDir, siteUrl, docsDir } = context;
     // Filter files based on include patterns
     let filteredFiles = allFiles;
     if (includePatterns.length > 0) {
         filteredFiles = allFiles.filter(file => {
-            const relativePath = path.relative(siteDir, file);
-            return includePatterns.some(pattern => (0, minimatch_1.minimatch)(relativePath, pattern, { matchBase: true }));
+            return includePatterns.some(pattern => matchesPattern(file, pattern, siteDir, docsDir));
         });
     }
     // Apply ignore patterns
     if (ignorePatterns.length > 0) {
         filteredFiles = filteredFiles.filter(file => {
-            const relativePath = path.relative(siteDir, file);
-            return !ignorePatterns.some(pattern => (0, minimatch_1.minimatch)(relativePath, pattern, { matchBase: true }));
+            return !ignorePatterns.some(pattern => matchesPattern(file, pattern, siteDir, docsDir));
         });
     }
     // Order files according to orderPatterns
@@ -198,8 +383,7 @@ async function processFilesWithPatterns(context, allFiles, includePatterns = [],
         // Process files according to orderPatterns
         for (const pattern of orderPatterns) {
             const matchingFiles = filteredFiles.filter(file => {
-                const relativePath = path.relative(siteDir, file);
-                return (0, minimatch_1.minimatch)(relativePath, pattern, { matchBase: true }) && !matchedFiles.has(file);
+                return matchesPattern(file, pattern, siteDir, docsDir) && !matchedFiles.has(file);
             });
             for (const file of matchingFiles) {
                 filesToProcess.push(file);
@@ -215,9 +399,8 @@ async function processFilesWithPatterns(context, allFiles, includePatterns = [],
     else {
         filesToProcess = filteredFiles;
     }
-    // Process each file to generate DocInfo
-    const processedDocs = [];
-    for (const filePath of filesToProcess) {
+    // Process files in parallel using Promise.allSettled
+    const results = await Promise.allSettled(filesToProcess.map(async (filePath) => {
         try {
             // Determine if this is a blog or docs file
             const isBlogFile = filePath.includes(path.join(siteDir, 'blog'));
@@ -225,73 +408,27 @@ async function processFilesWithPatterns(context, allFiles, includePatterns = [],
             const baseDir = siteDir;
             const pathPrefix = isBlogFile ? 'blog' : 'docs';
             // Try to find the resolved URL for this file from the route map
-            let resolvedUrl;
-            if (context.routeMap) {
-                // Convert file path to a potential route path
-                const relativePath = path.relative(baseDir, filePath)
-                    .replace(/\\/g, '/')
+            const resolvedUrl = resolveDocumentUrl(filePath, baseDir, pathPrefix, context);
+            // Log when we successfully resolve a URL using Docusaurus routes
+            if (resolvedUrl && context.routeMap) {
+                const relativePath = (0, utils_1.normalizePath)(path.relative(baseDir, filePath))
                     .replace(/\.mdx?$/, '')
                     .replace(/\/index$/, '');
-                // Function to remove numbered prefixes from path segments
-                const removeNumberedPrefixes = (path) => {
-                    return path.split('/').map(segment => {
-                        // Remove numbered prefixes like "01-", "1-", "001-" from each segment
-                        return segment.replace(/^\d+-/, '');
-                    }).join('/');
-                };
-                // Check various possible route patterns
-                const cleanPath = removeNumberedPrefixes(relativePath);
-                const possiblePaths = [
-                    `/${pathPrefix}/${cleanPath}`,
-                    `/${cleanPath}`,
-                    `/${pathPrefix}/${relativePath}`, // Try with original path
-                    `/${relativePath}`, // Try without prefix
-                ];
-                // Also handle nested folder structures with numbered prefixes
-                const segments = relativePath.split('/');
-                if (segments.length > 1) {
-                    // Try removing numbered prefixes from different levels
-                    for (let i = 0; i < segments.length; i++) {
-                        const modifiedSegments = [...segments];
-                        modifiedSegments[i] = modifiedSegments[i].replace(/^\d+-/, '');
-                        const modifiedPath = modifiedSegments.join('/');
-                        possiblePaths.push(`/${pathPrefix}/${modifiedPath}`);
-                        possiblePaths.push(`/${modifiedPath}`);
-                    }
-                }
-                // Try to find a match in the route map
-                for (const possiblePath of possiblePaths) {
-                    if (context.routeMap.has(possiblePath)) {
-                        resolvedUrl = context.routeMap.get(possiblePath);
-                        break;
-                    }
-                }
-                // If still not found, try to find the best match using the routesPaths array
-                if (!resolvedUrl && context.routesPaths) {
-                    const normalizedCleanPath = cleanPath.toLowerCase();
-                    const matchingRoute = context.routesPaths.find(routePath => {
-                        const normalizedRoute = routePath.toLowerCase();
-                        return normalizedRoute.endsWith(`/${normalizedCleanPath}`) ||
-                            normalizedRoute === `/${pathPrefix}/${normalizedCleanPath}` ||
-                            normalizedRoute === `/${normalizedCleanPath}`;
-                    });
-                    if (matchingRoute) {
-                        resolvedUrl = matchingRoute;
-                    }
-                }
-                // Log when we successfully resolve a URL using Docusaurus routes
-                if (resolvedUrl && resolvedUrl !== `/${pathPrefix}/${relativePath}`) {
-                    console.log(`Resolved URL for ${path.basename(filePath)}: ${resolvedUrl} (was: /${pathPrefix}/${relativePath})`);
+                if (resolvedUrl !== `/${pathPrefix}/${relativePath}`) {
+                    utils_1.logger.verbose(`Resolved URL for ${path.basename(filePath)}: ${resolvedUrl} (was: /${pathPrefix}/${relativePath})`);
                 }
             }
             const docInfo = await processMarkdownFile(filePath, baseDir, siteUrl, pathPrefix, context.options.pathTransformation, context.options.excludeImports || false, context.options.removeDuplicateHeadings || false, resolvedUrl);
-            if (docInfo !== null) {
-                processedDocs.push(docInfo);
-            }
+            return docInfo;
         }
         catch (err) {
-            console.warn(`Error processing ${filePath}: ${err.message}`);
+            utils_1.logger.warn(`Error processing ${filePath}: ${(0, utils_1.getErrorMessage)(err)}`);
+            return null;
         }
-    }
+    }));
+    // Filter successful results and non-null DocInfo objects
+    const processedDocs = results
+        .filter((r) => r.status === 'fulfilled' && r.value !== null)
+        .map(r => r.value);
     return processedDocs;
 }

package/lib/types.d.ts

@@ -87,6 +87,16 @@ export interface PluginOptions {
     rootContent?: string;
     /** Custom content to include at the root level of llms-full.txt (after title/description, before content sections) */
     fullRootContent?: string;
+    /** Whether to preserve directory structure in generated markdown files (default: true) */
+    preserveDirectoryStructure?: boolean;
+    /** Batch size for processing large document sets to prevent memory issues (default: 100) */
+    processingBatchSize?: number;
+    /** Logging level for plugin output (default: 'normal'). Options: 'quiet', 'normal', 'verbose' */
+    logLevel?: 'quiet' | 'normal' | 'verbose';
+    /** Whether to warn about files that are ignored (no extension or unsupported extension) (default: false) */
+    warnOnIgnoredFiles?: boolean;
+    /** Index signature for Docusaurus plugin compatibility */
+    [key: string]: unknown;
 }
 /**
  * Plugin context with processed options

package/lib/utils.d.ts

@@ -2,6 +2,129 @@
  * Utility functions for the docusaurus-plugin-llms plugin
  */
 import { PluginOptions } from './types';
+/**
+ * Null/Undefined Handling Guidelines:
+ *
+ * 1. For required parameters: Throw early if null/undefined
+ * 2. For optional parameters: Use optional chaining `value?.property`
+ * 3. For explicit null checks: Use `!== null` and `!== undefined` or the isDefined type guard
+ * 4. For string validation: Use isNonEmptyString() type guard
+ * 5. For truthy checks on booleans: Use explicit comparison or Boolean(value)
+ *
+ * Avoid: `if (value)` when value could be 0, '', or false legitimately
+ * Use: Type guards for consistent, type-safe checks
+ */
+/**
+ * Type guard to check if a value is defined (not null or undefined)
+ * @param value - Value to check
+ * @returns True if value is not null or undefined
+ */
+export declare function isDefined<T>(value: T | null | undefined): value is T;
+/**
+ * Type guard to check if a value is a non-empty string
+ * @param value - Value to check
+ * @returns True if value is a string with at least one non-whitespace character
+ */
+export declare function isNonEmptyString(value: unknown): value is string;
+/**
+ * Type guard to check if a value is a non-empty array
+ * @param value - Value to check
+ * @returns True if value is an array with at least one element
+ */
+export declare function isNonEmptyArray<T>(value: unknown): value is T[];
+/**
+ * Safely extract an error message from an unknown error value
+ * @param error - The error value (can be Error, string, or any other type)
+ * @returns A string representation of the error
+ */
+export declare function getErrorMessage(error: unknown): string;
+/**
+ * Extract stack trace from unknown error types
+ * @param error - The error value (can be Error or any other type)
+ * @returns Stack trace if available, undefined otherwise
+ */
+export declare function getErrorStack(error: unknown): string | undefined;
+/**
+ * Custom error class for validation errors
+ */
+export declare class ValidationError extends Error {
+    constructor(message: string);
+}
+/**
+ * Validates that a value is not null or undefined
+ * @param value - The value to validate
+ * @param paramName - The parameter name for error messages
+ * @returns The validated value
+ * @throws ValidationError if the value is null or undefined
+ */
+export declare function validateRequired<T>(value: T | null | undefined, paramName: string): T;
+/**
+ * Validates that a value is a string and optionally checks its properties
+ * @param value - The value to validate
+ * @param paramName - The parameter name for error messages
+ * @param options - Validation options for min/max length and pattern
+ * @returns The validated string
+ * @throws ValidationError if validation fails
+ */
+export declare function validateString(value: unknown, paramName: string, options?: {
+    minLength?: number;
+    maxLength?: number;
+    pattern?: RegExp;
+}): string;
+/**
+ * Validates that a value is an array and optionally validates elements
+ * @param value - The value to validate
+ * @param paramName - The parameter name for error messages
+ * @param elementValidator - Optional function to validate each element
+ * @returns The validated array
+ * @throws ValidationError if validation fails
+ */
+export declare function validateArray<T>(value: unknown, paramName: string, elementValidator?: (item: unknown) => boolean): T[];
+/**
+ * Logging level enumeration
+ */
+export declare enum LogLevel {
+    QUIET = 0,
+    NORMAL = 1,
+    VERBOSE = 2
+}
+/**
+ * Set the logging level for the plugin
+ * @param level - The logging level to use
+ */
+export declare function setLogLevel(level: LogLevel): void;
+/**
+ * Logger utility for consistent logging across the plugin
+ */
+export declare const logger: {
+    error: (message: string) => void;
+    warn: (message: string) => void;
+    info: (message: string) => void;
+    verbose: (message: string) => void;
+};
+/**
+ * Normalizes a file path by converting all backslashes to forward slashes.
+ * This ensures consistent path handling across Windows and Unix systems.
+ *
+ * @param filePath - The file path to normalize
+ * @returns The normalized path with forward slashes
+ * @throws ValidationError if filePath is not a string
+ */
+export declare function normalizePath(filePath: string): string;
+/**
+ * Validates that a file path does not exceed the platform-specific maximum length
+ * @param filePath - The file path to validate
+ * @returns True if the path is within limits, false otherwise
+ */
+export declare function validatePathLength(filePath: string): boolean;
+/**
+ * Shortens a file path by creating a hash-based filename if the path is too long
+ * @param fullPath - The full file path that may be too long
+ * @param outputDir - The output directory base path
+ * @param relativePath - The relative path from the output directory
+ * @returns A shortened path if necessary, or the original path if it's within limits
+ */
+export declare function shortenPathIfNeeded(fullPath: string, outputDir: string, relativePath: string): string;
 /**
  * Write content to a file
  * @param filePath - Path to write the file to
@@ -11,25 +134,30 @@ export declare function writeFile(filePath: string, data: string): Promise<void>
 /**
  * Read content from a file
  * @param filePath - Path of the file to read
- * @returns Content of the file
+ * @returns Content of the file with BOM removed if present
  */
 export declare function readFile(filePath: string): Promise<string>;
 /**
  * Check if a file should be ignored based on glob patterns
+ * Matches against both site-relative and docs-relative paths
  * @param filePath - Path to the file
- * @param baseDir - Base directory for relative paths
+ * @param baseDir - Base directory (site root) for relative paths
  * @param ignorePatterns - Glob patterns for files to ignore
+ * @param docsDir - Docs directory name (e.g., 'docs')
  * @returns Whether the file should be ignored
  */
-export declare function shouldIgnoreFile(filePath: string, baseDir: string, ignorePatterns: string[]): boolean;
+export declare function shouldIgnoreFile(filePath: string, baseDir: string, ignorePatterns: string[], docsDir?: string): boolean;
 /**
  * Recursively reads all Markdown files in a directory
  * @param dir - Directory to scan
- * @param baseDir - Base directory for relative paths
+ * @param baseDir - Base directory (site root) for relative paths
  * @param ignorePatterns - Glob patterns for files to ignore
+ * @param docsDir - Docs directory name (e.g., 'docs')
+ * @param warnOnIgnoredFiles - Whether to warn about ignored files
+ * @param visitedPaths - Set of already visited real paths to detect symlink loops (internal use)
  * @returns Array of file paths
  */
-export declare function readMarkdownFiles(dir: string, baseDir: string, ignorePatterns?: string[]): Promise<string[]>;
+export declare function readMarkdownFiles(dir: string, baseDir: string, ignorePatterns?: string[], docsDir?: string, warnOnIgnoredFiles?: boolean, visitedPaths?: Set<string>): Promise<string[]>;
 /**
  * Extract title from content or use the filename
  * @param data - Frontmatter data
@@ -42,9 +170,10 @@ export declare function extractTitle(data: any, content: string, filePath: strin
  * Resolve and inline partial imports in markdown content
  * @param content - The markdown content with import statements
  * @param filePath - The path of the file containing the imports
+ * @param importChain - Set of file paths in the current import chain (for circular dependency detection)
  * @returns Content with partials resolved
  */
-export declare function resolvePartialImports(content: string, filePath: string): Promise<string>;
+export declare function resolvePartialImports(content: string, filePath: string, importChain?: Set<string>): Promise<string>;
 /**
  * Clean markdown content for LLM consumption
  * @param content - Raw markdown content
@@ -65,14 +194,19 @@ export declare function applyPathTransformations(urlPath: string, pathTransforma
  * @param input - Input string (typically a title)
  * @param fallback - Fallback string if input becomes empty after sanitization
  * @returns Sanitized filename (without extension)
+ * @throws ValidationError if input or fallback are not strings
  */
-export declare function sanitizeForFilename(input: string, fallback?: string): string;
+export declare function sanitizeForFilename(input: string, fallback?: string, options?: {
+    preserveUnicode?: boolean;
+    preserveCase?: boolean;
+}): string;
 /**
  * Ensure a unique identifier from a set of used identifiers
  * @param baseIdentifier - Base identifier to make unique
  * @param usedIdentifiers - Set of already used identifiers
  * @param suffix - Suffix pattern (default: number in parentheses)
  * @returns Unique identifier
+ * @throws ValidationError if baseIdentifier is not a string or usedIdentifiers is not a Set
  */
 export declare function ensureUniqueIdentifier(baseIdentifier: string, usedIdentifiers: Set<string>, suffix?: (counter: number, base: string) => string): string;
 /**