docusaurus-plugin-llms 0.2.0 → 0.3.0

package/src/processor.ts

@@ -6,12 +6,16 @@ import * as path from 'path';
 import matter from 'gray-matter';
 import { minimatch } from 'minimatch';
 import { DocInfo, PluginContext } from './types';
-import { 
-  readFile, 
-  extractTitle, 
-  cleanMarkdownContent, 
+import {
+  readFile,
+  extractTitle,
+  cleanMarkdownContent,
   applyPathTransformations,
-  resolvePartialImports
+  resolvePartialImports,
+  normalizePath,
+  logger,
+  getErrorMessage,
+  isNonEmptyString
 } from './utils';
 
 /**
@@ -24,8 +28,8 @@ import {
  * @returns Processed file data
  */
 export async function processMarkdownFile(
-  filePath: string, 
-  baseDir: string, 
+  filePath: string,
+  baseDir: string,
   siteUrl: string,
   pathPrefix: string = 'docs',
   pathTransformation?: {
@@ -38,35 +42,70 @@ export async function processMarkdownFile(
 ): Promise<DocInfo | null> {
   const content = await readFile(filePath);
   const { data, content: markdownContent } = matter(content);
-  
+
   // Skip draft files
   if (data.draft === true) {
     return null;
   }
+
+  // Validate and clean empty frontmatter fields
+  // Empty strings should be treated as undefined to allow fallback logic
+  if (data.title !== undefined && !isNonEmptyString(data.title)) {
+    logger.warn(`Empty title in frontmatter for ${filePath}. Using fallback.`);
+    data.title = undefined;
+  }
+
+  if (data.description !== undefined && !isNonEmptyString(data.description)) {
+    data.description = undefined;
+  }
+
+  if (data.slug !== undefined && !isNonEmptyString(data.slug)) {
+    data.slug = undefined;
+  }
+
+  if (data.id !== undefined && !isNonEmptyString(data.id)) {
+    data.id = undefined;
+  }
   
   // Resolve partial imports before processing
   const resolvedContent = await resolvePartialImports(markdownContent, filePath);
   
   const relativePath = path.relative(baseDir, filePath);
   // Convert to URL path format (replace backslashes with forward slashes on Windows)
-  const normalizedPath = relativePath.replace(/\\/g, '/');
+  const normalizedPath = normalizePath(relativePath);
   
   let fullUrl: string;
-  
-  if (resolvedUrl) {
+
+  if (isNonEmptyString(resolvedUrl)) {
     // Use the actual resolved URL from Docusaurus if provided
-    fullUrl = new URL(resolvedUrl, siteUrl).toString();
+    try {
+      fullUrl = new URL(resolvedUrl, siteUrl).toString();
+    } catch (error: unknown) {
+      logger.warn(`Invalid URL construction: ${resolvedUrl} with base ${siteUrl}. Using fallback.`);
+      // Fallback to string concatenation with proper path joining
+      const baseUrl = siteUrl.endsWith('/') ? siteUrl.slice(0, -1) : siteUrl;
+      const urlPath = resolvedUrl.startsWith('/') ? resolvedUrl : `/${resolvedUrl}`;
+      fullUrl = baseUrl + urlPath;
+    }
   } else {
     // Fallback to the old path construction method
     // Convert .md extension to appropriate path
     const linkPathBase = normalizedPath.replace(/\.mdx?$/, '');
     
     // Handle index files specially
-    const linkPath = linkPathBase.endsWith('index') 
+    let linkPath = linkPathBase.endsWith('index') 
       ? linkPathBase.replace(/\/index$/, '') 
       : linkPathBase;
     
-    // Apply path transformations to the link path
+    // linkPath might include the pathPrefix (e.g., "docs/api/core")
+    // We need to remove the pathPrefix before applying transformations, then add it back later
+    if (pathPrefix && linkPath.startsWith(`${pathPrefix}/`)) {
+      linkPath = linkPath.substring(`${pathPrefix}/`.length);
+    } else if (pathPrefix && linkPath === pathPrefix) {
+      linkPath = '';
+    }
+    
+    // Apply path transformations to the clean link path (without pathPrefix)
     const transformedLinkPath = applyPathTransformations(linkPath, pathTransformation);
     
     // Also apply path transformations to the pathPrefix if it's not empty
@@ -76,13 +115,44 @@ export async function processMarkdownFile(
       transformedPathPrefix = '';
     }
     
-    // Generate full URL with transformed path and path prefix
-    fullUrl = new URL(
-      `${transformedPathPrefix ? `${transformedPathPrefix}/` : ''}${transformedLinkPath}`, 
-      siteUrl
-    ).toString();
+    // Ensure path segments are URL-safe with sophisticated encoding detection
+    const encodedLinkPath = transformedLinkPath.split('/').map(segment => {
+      // Check if segment contains characters that need encoding
+      // Unreserved characters (per RFC 3986): A-Z a-z 0-9 - . _ ~
+      if (!/[^A-Za-z0-9\-._~]/.test(segment)) {
+        // Segment only contains unreserved characters, no encoding needed
+        return segment;
+      }
+
+      try {
+        // Try to decode - if it changes, it was already encoded
+        const decoded = decodeURIComponent(segment);
+        if (decoded !== segment) {
+          // Was already encoded, return as-is
+          return segment;
+        }
+        // Not encoded, encode it
+        return encodeURIComponent(segment);
+      } catch {
+        // Malformed encoding, re-encode
+        return encodeURIComponent(segment);
+      }
+    }).join('/');
+
+    // Construct URL by encoding path components, then combine with site URL
+    // We don't use URL constructor for the full path because it decodes some characters
+    const pathPart = transformedPathPrefix ? `${transformedPathPrefix}/${encodedLinkPath}` : encodedLinkPath;
+    try {
+      const baseUrl = new URL(siteUrl);
+      fullUrl = `${baseUrl.origin}/${pathPart}`;
+    } catch (error: unknown) {
+      logger.warn(`Invalid siteUrl: ${siteUrl}. Using fallback.`);
+      // Fallback to string concatenation with proper path joining
+      const baseUrl = siteUrl.endsWith('/') ? siteUrl.slice(0, -1) : siteUrl;
+      fullUrl = `${baseUrl}/${pathPart}`;
+    }
   }
-  
+
   // Extract title
   const title = extractTitle(data, resolvedContent, filePath);
   
@@ -90,7 +160,7 @@ export async function processMarkdownFile(
   let description = '';
   
   // First priority: Use frontmatter description if available
-  if (data.description) {
+  if (isNonEmptyString(data.description)) {
     description = data.description;
   } else {
     // Second priority: Find the first non-heading paragraph
@@ -115,14 +185,14 @@ export async function processMarkdownFile(
   
   // Only remove heading markers at the beginning of descriptions or lines
   // This preserves # characters that are part of the content
-  if (description) {
+  if (isNonEmptyString(description)) {
     // Original approach had issues with hashtags inside content
     // Fix: Only remove # symbols at the beginning of lines or description
     // that are followed by a space (actual heading markers)
     description = description.replace(/^(#+)\s+/gm, '');
     
     // Special handling for description frontmatter with heading markers
-    if (data.description && data.description.startsWith('#')) {
+    if (isNonEmptyString(data.description) && data.description.startsWith('#')) {
       // If the description in frontmatter starts with a heading marker,
       // we should preserve it in the extracted description
       description = description.replace(/^#+\s+/, '');
@@ -133,17 +203,17 @@ export async function processMarkdownFile(
     
     // Validate that the description doesn't contain markdown headings
     if (description.match(/^#+\s+/m)) {
-      console.warn(`Warning: Description for "${title}" may still contain heading markers`);
+      logger.warn(`Warning: Description for "${title}" may still contain heading markers`);
     }
     
     // Warn if the description contains HTML tags
     if (/<[^>]+>/g.test(description)) {
-      console.warn(`Warning: Description for "${title}" contains HTML tags`);
+      logger.warn(`Warning: Description for "${title}" contains HTML tags`);
     }
     
     // Warn if the description is very long
     if (description.length > 500) {
-      console.warn(`Warning: Description for "${title}" is very long (${description.length} characters)`);
+      logger.warn(`Warning: Description for "${title}" is very long (${description.length} characters)`);
     }
   }
   
@@ -156,9 +226,147 @@ export async function processMarkdownFile(
     url: fullUrl,
     content: cleanedContent,
     description: description || '',
+    frontMatter: data,
   };
 }
 
+/**
+ * Remove numbered prefixes from path segments (e.g., "01-intro" -> "intro")
+ */
+function removeNumberedPrefixes(path: string): string {
+  return path.split('/').map(segment => {
+    // Remove numbered prefixes like "01-", "1-", "001-" from each segment
+    return segment.replace(/^\d+-/, '');
+  }).join('/');
+}
+
+/**
+ * Try to find a route in the route map from a list of possible paths
+ */
+function findRouteInMap(routeMap: Map<string, string>, possiblePaths: string[]): string | undefined {
+  for (const possiblePath of possiblePaths) {
+    const route = routeMap.get(possiblePath) || routeMap.get(possiblePath + '/');
+    if (route) {
+      return route;
+    }
+  }
+  return undefined;
+}
+
+/**
+ * Try exact match for route resolution
+ */
+function tryExactRouteMatch(
+  routeMap: Map<string, string>,
+  relativePath: string,
+  pathPrefix: string
+): string | undefined {
+  const possiblePaths = [
+    `/${pathPrefix}/${relativePath}`,
+    `/${relativePath}`,
+  ];
+  return findRouteInMap(routeMap, possiblePaths);
+}
+
+/**
+ * Try route resolution with numbered prefix removal
+ */
+function tryNumberedPrefixResolution(
+  routeMap: Map<string, string>,
+  relativePath: string,
+  pathPrefix: string
+): string | undefined {
+  const cleanPath = removeNumberedPrefixes(relativePath);
+
+  // Try basic cleaned path
+  const basicPaths = [`/${pathPrefix}/${cleanPath}`, `/${cleanPath}`];
+  const basicMatch = findRouteInMap(routeMap, basicPaths);
+  if (basicMatch) {
+    return basicMatch;
+  }
+
+  // Try nested folder structures with numbered prefixes at different levels
+  const segments = relativePath.split('/');
+  if (segments.length > 1) {
+    for (let i = 0; i < segments.length; i++) {
+      const modifiedSegments = [...segments];
+      modifiedSegments[i] = modifiedSegments[i].replace(/^\d+-/, '');
+      const modifiedPath = modifiedSegments.join('/');
+      const pathsToTry = [`/${pathPrefix}/${modifiedPath}`, `/${modifiedPath}`];
+
+      const match = findRouteInMap(routeMap, pathsToTry);
+      if (match) {
+        return match;
+      }
+    }
+  }
+
+  return undefined;
+}
+
+/**
+ * Try finding best match using routes paths array
+ */
+function tryRoutesPathsMatch(
+  routesPaths: string[],
+  relativePath: string,
+  pathPrefix: string
+): string | undefined {
+  const cleanPath = removeNumberedPrefixes(relativePath);
+  const normalizedCleanPath = cleanPath.toLowerCase();
+
+  return routesPaths.find(routePath => {
+    const normalizedRoute = routePath.toLowerCase();
+    return normalizedRoute.endsWith(`/${normalizedCleanPath}`) ||
+           normalizedRoute === `/${pathPrefix}/${normalizedCleanPath}` ||
+           normalizedRoute === `/${normalizedCleanPath}`;
+  });
+}
+
+/**
+ * Resolve the URL for a document using Docusaurus routes
+ * @param filePath - Full path to the file
+ * @param baseDir - Base directory (typically siteDir)
+ * @param pathPrefix - Path prefix ('docs' or 'blog')
+ * @param context - Plugin context with route map
+ * @returns Resolved URL or undefined if not found
+ */
+function resolveDocumentUrl(
+  filePath: string,
+  baseDir: string,
+  pathPrefix: string,
+  context: PluginContext
+): string | undefined {
+  // Early return if no route map available
+  if (!context.routeMap) {
+    return undefined;
+  }
+
+  // Convert file path to a potential route path
+  const relativePath = normalizePath(path.relative(baseDir, filePath))
+    .replace(/\.mdx?$/, '')
+    .replace(/\/index$/, '');
+
+  // Try exact match first (respects Docusaurus's resolved routes)
+  const exactMatch = tryExactRouteMatch(context.routeMap, relativePath, pathPrefix);
+  if (exactMatch) {
+    return exactMatch;
+  }
+
+  // Try numbered prefix removal as fallback
+  const prefixMatch = tryNumberedPrefixResolution(context.routeMap, relativePath, pathPrefix);
+  if (prefixMatch) {
+    return prefixMatch;
+  }
+
+  // Try to find the best match using the routesPaths array
+  if (context.routesPaths) {
+    return tryRoutesPathsMatch(context.routesPaths, relativePath, pathPrefix);
+  }
+
+  return undefined;
+}
+
 /**
  * Process files based on include patterns, ignore patterns, and ordering
  * @param context - Plugin context
@@ -169,6 +377,37 @@ export async function processMarkdownFile(
  * @param includeUnmatched - Whether to include unmatched files
  * @returns Processed files
  */
+/**
+ * Helper function to check if a file matches a pattern
+ * Tries matching against multiple path variants for better usability
+ */
+function matchesPattern(file: string, pattern: string, siteDir: string, docsDir: string): boolean {
+  const minimatchOptions = { matchBase: true };
+
+  // Get site-relative path (e.g., "docs/quickstart/file.md")
+  const siteRelativePath = normalizePath(path.relative(siteDir, file));
+
+  // Get docs-relative path (e.g., "quickstart/file.md")
+  // Normalize both paths to handle different path separators and resolve any .. or .
+  const docsBaseDir = path.resolve(path.join(siteDir, docsDir));
+  const resolvedFile = path.resolve(file);
+  const docsRelativePath = resolvedFile.startsWith(docsBaseDir)
+    ? normalizePath(path.relative(docsBaseDir, resolvedFile))
+    : null;
+
+  // Try matching against site-relative path
+  if (minimatch(siteRelativePath, pattern, minimatchOptions)) {
+    return true;
+  }
+
+  // Try matching against docs-relative path if available
+  if (docsRelativePath && minimatch(docsRelativePath, pattern, minimatchOptions)) {
+    return true;
+  }
+
+  return false;
+}
+
 export async function processFilesWithPatterns(
   context: PluginContext,
   allFiles: string[],
@@ -184,9 +423,8 @@ export async function processFilesWithPatterns(
   
   if (includePatterns.length > 0) {
     filteredFiles = allFiles.filter(file => {
-      const relativePath = path.relative(siteDir, file);
-      return includePatterns.some(pattern => 
-        minimatch(relativePath, pattern, { matchBase: true })
+      return includePatterns.some(pattern =>
+        matchesPattern(file, pattern, siteDir, docsDir)
       );
     });
   }
@@ -194,9 +432,8 @@ export async function processFilesWithPatterns(
   // Apply ignore patterns
   if (ignorePatterns.length > 0) {
     filteredFiles = filteredFiles.filter(file => {
-      const relativePath = path.relative(siteDir, file);
-      return !ignorePatterns.some(pattern => 
-        minimatch(relativePath, pattern, { matchBase: true })
+      return !ignorePatterns.some(pattern =>
+        matchesPattern(file, pattern, siteDir, docsDir)
       );
     });
   }
@@ -210,8 +447,7 @@ export async function processFilesWithPatterns(
     // Process files according to orderPatterns
     for (const pattern of orderPatterns) {
       const matchingFiles = filteredFiles.filter(file => {
-        const relativePath = path.relative(siteDir, file);
-        return minimatch(relativePath, pattern, { matchBase: true }) && !matchedFiles.has(file);
+        return matchesPattern(file, pattern, siteDir, docsDir) && !matchedFiles.has(file);
       });
       
       for (const file of matchingFiles) {
@@ -229,100 +465,51 @@ export async function processFilesWithPatterns(
     filesToProcess = filteredFiles;
   }
   
-  // Process each file to generate DocInfo
-  const processedDocs: DocInfo[] = [];
-  
-  for (const filePath of filesToProcess) {
-    try {
-      // Determine if this is a blog or docs file
-      const isBlogFile = filePath.includes(path.join(siteDir, 'blog'));
-      const baseDir = isBlogFile ? path.join(siteDir, 'blog') : path.join(siteDir, docsDir);
-      const pathPrefix = isBlogFile ? 'blog' : 'docs';
-      
-      // Try to find the resolved URL for this file from the route map
-      let resolvedUrl: string | undefined;
-      if (context.routeMap) {
-        // Convert file path to a potential route path
-        const relativePath = path.relative(baseDir, filePath)
-          .replace(/\\/g, '/')
-          .replace(/\.mdx?$/, '')
-          .replace(/\/index$/, '');
-        
-        // Function to remove numbered prefixes from path segments
-        const removeNumberedPrefixes = (path: string): string => {
-          return path.split('/').map(segment => {
-            // Remove numbered prefixes like "01-", "1-", "001-" from each segment
-            return segment.replace(/^\d+-/, '');
-          }).join('/');
-        };
-        
-        // Check various possible route patterns
-        const cleanPath = removeNumberedPrefixes(relativePath);
-        const possiblePaths = [
-          `/${pathPrefix}/${cleanPath}`,
-          `/${cleanPath}`,
-          `/${pathPrefix}/${relativePath}`, // Try with original path
-          `/${relativePath}`, // Try without prefix
-        ];
-        
-        // Also handle nested folder structures with numbered prefixes
-        const segments = relativePath.split('/');
-        if (segments.length > 1) {
-          // Try removing numbered prefixes from different levels
-          for (let i = 0; i < segments.length; i++) {
-            const modifiedSegments = [...segments];
-            modifiedSegments[i] = modifiedSegments[i].replace(/^\d+-/, '');
-            const modifiedPath = modifiedSegments.join('/');
-            possiblePaths.push(`/${pathPrefix}/${modifiedPath}`);
-            possiblePaths.push(`/${modifiedPath}`);
-          }
-        }
-        
-        // Try to find a match in the route map
-        for (const possiblePath of possiblePaths) {
-          if (context.routeMap.has(possiblePath)) {
-            resolvedUrl = context.routeMap.get(possiblePath);
-            break;
-          }
-        }
-        
-        // If still not found, try to find the best match using the routesPaths array
-        if (!resolvedUrl && context.routesPaths) {
-          const normalizedCleanPath = cleanPath.toLowerCase();
-          const matchingRoute = context.routesPaths.find(routePath => {
-            const normalizedRoute = routePath.toLowerCase();
-            return normalizedRoute.endsWith(`/${normalizedCleanPath}`) || 
-                   normalizedRoute === `/${pathPrefix}/${normalizedCleanPath}` ||
-                   normalizedRoute === `/${normalizedCleanPath}`;
-          });
-          if (matchingRoute) {
-            resolvedUrl = matchingRoute;
-          }
-        }
-        
+  // Process files in parallel using Promise.allSettled
+  const results = await Promise.allSettled(
+    filesToProcess.map(async (filePath) => {
+      try {
+        // Determine if this is a blog or docs file
+        const isBlogFile = filePath.includes(path.join(siteDir, 'blog'));
+        // Use siteDir as baseDir to preserve full directory structure (docs/path/file.md instead of just path/file.md)
+        const baseDir = siteDir;
+        const pathPrefix = isBlogFile ? 'blog' : 'docs';
+
+        // Try to find the resolved URL for this file from the route map
+        const resolvedUrl = resolveDocumentUrl(filePath, baseDir, pathPrefix, context);
+
         // Log when we successfully resolve a URL using Docusaurus routes
-        if (resolvedUrl && resolvedUrl !== `/${pathPrefix}/${relativePath}`) {
-          console.log(`Resolved URL for ${path.basename(filePath)}: ${resolvedUrl} (was: /${pathPrefix}/${relativePath})`);
+        if (resolvedUrl && context.routeMap) {
+          const relativePath = normalizePath(path.relative(baseDir, filePath))
+            .replace(/\.mdx?$/, '')
+            .replace(/\/index$/, '');
+          if (resolvedUrl !== `/${pathPrefix}/${relativePath}`) {
+            logger.verbose(`Resolved URL for ${path.basename(filePath)}: ${resolvedUrl} (was: /${pathPrefix}/${relativePath})`);
+          }
         }
+
+        const docInfo = await processMarkdownFile(
+          filePath,
+          baseDir,
+          siteUrl,
+          pathPrefix,
+          context.options.pathTransformation,
+          context.options.excludeImports || false,
+          context.options.removeDuplicateHeadings || false,
+          resolvedUrl
+        );
+        return docInfo;
+      } catch (err: unknown) {
+        logger.warn(`Error processing ${filePath}: ${getErrorMessage(err)}`);
+        return null;
       }
-      
-      const docInfo = await processMarkdownFile(
-        filePath, 
-        baseDir, 
-        siteUrl,
-        pathPrefix,
-        context.options.pathTransformation,
-        context.options.excludeImports || false,
-        context.options.removeDuplicateHeadings || false,
-        resolvedUrl
-      );
-      if (docInfo !== null) {
-        processedDocs.push(docInfo);
-      }
-    } catch (err: any) {
-      console.warn(`Error processing ${filePath}: ${err.message}`);
-    }
-  }
-  
+    })
+  );
+
+  // Filter successful results and non-null DocInfo objects
+  const processedDocs = results
+    .filter((r): r is PromiseFulfilledResult<DocInfo | null> => r.status === 'fulfilled' && r.value !== null)
+    .map(r => r.value as DocInfo);
+
   return processedDocs;
-} 
+} 

package/src/types.ts

@@ -13,6 +13,7 @@ export interface DocInfo {
   url: string;
   content: string;
   description: string;
+  frontMatter?: Record<string, any>;
 }
 
 /**
@@ -109,12 +110,30 @@ export interface PluginOptions {
   
   /** Whether to generate individual markdown files and link to them from llms.txt instead of original docs (default: false) */
   generateMarkdownFiles?: boolean;
-  
+
+  /** Array of frontmatter keys to preserve in generated individual markdown files (only used when generateMarkdownFiles is true) */
+  keepFrontMatter?: string[];
+
   /** Custom content to include at the root level of llms.txt (after title/description, before TOC) */
   rootContent?: string;
   
   /** Custom content to include at the root level of llms-full.txt (after title/description, before content sections) */
   fullRootContent?: string;
+
+  /** Whether to preserve directory structure in generated markdown files (default: true) */
+  preserveDirectoryStructure?: boolean;
+
+  /** Batch size for processing large document sets to prevent memory issues (default: 100) */
+  processingBatchSize?: number;
+
+  /** Logging level for plugin output (default: 'normal'). Options: 'quiet', 'normal', 'verbose' */
+  logLevel?: 'quiet' | 'normal' | 'verbose';
+
+  /** Whether to warn about files that are ignored (no extension or unsupported extension) (default: false) */
+  warnOnIgnoredFiles?: boolean;
+
+  /** Index signature for Docusaurus plugin compatibility */
+  [key: string]: unknown;
 }
 
 /**