@clidey/dory 0.47.0 → 0.49.0

package/src/plugins/llm-txt-generator.ts

@@ -1,193 +1,154 @@
 import type { Plugin } from 'vite';
 import fs from 'fs';
 import path from 'path';
+import {
+  readDoryConfig,
+  getNavigationOrder,
+  parseFrontmatter,
+  findMdxFilesMap,
+  pathFromFilename,
+} from './shared';
 
 interface LlmTxtGeneratorOptions {
   docsDir?: string;
-  outputPath?: string;
 }
 
-// Read dory.json to get navigation order (flat list, in order)
-function getNavigationOrder(docsDir: string): string[] {
-  const doryJsonPath = path.join(docsDir, 'dory.json');
-  if (!fs.existsSync(doryJsonPath)) return [];
-  let doryJson;
-  try {
-    doryJson = JSON.parse(fs.readFileSync(doryJsonPath, 'utf-8'));
-  } catch (error) {
-    const detail = error instanceof SyntaxError ? error.message : String(error);
-    console.error(`Failed to parse ${doryJsonPath}: ${detail}`);
-    return [];
-  }
-  const order: string[] = [];
-
-  function walkPages(pages: any[]) {
-    for (const page of pages) {
-      if (typeof page === 'string') {
-        order.push(page);
-      } else if (typeof page === 'object' && page.pages) {
-        walkPages(page.pages);
-      }
-    }
-  }
-
-  if (doryJson.navigation && Array.isArray(doryJson.navigation.tabs)) {
-    for (const tab of doryJson.navigation.tabs) {
-      if (tab.groups && Array.isArray(tab.groups)) {
-        for (const group of tab.groups) {
-          if (group.pages && Array.isArray(group.pages)) {
-            walkPages(group.pages);
-          }
-        }
-      }
-    }
-  }
-  return order;
-}
-
-// Function to parse frontmatter from MDX content
-function parseFrontmatter(content: string) {
-  const frontmatterMatch = content.match(/^---\n([\s\S]*?)\n---\n/);
-  if (!frontmatterMatch) return { data: {}, content };
-
-  const frontmatter = frontmatterMatch[1];
-  const data: Record<string, string> = {};
-
-  frontmatter.split('\n').forEach(line => {
-    const [key, ...valueParts] = line.split(':');
-    if (key && valueParts.length) {
-      const value = valueParts.join(':').trim();
-      data[key.trim()] = value.replace(/^["']|["']$/g, '');
-    }
-  });
-
-  return { data, content: content.slice(frontmatterMatch[0].length) };
-}
-
-// Function to recursively find all MDX files and map by relative path (without .mdx)
-function findMdxFilesMap(dir: string, baseDir: string): Record<string, string> {
-  const files: Record<string, string> = {};
-  const items = fs.readdirSync(dir, { withFileTypes: true });
-
-  for (const item of items) {
-    const fullPath = path.join(dir, item.name);
-    if (item.isDirectory()) {
-      Object.assign(files, findMdxFilesMap(fullPath, baseDir));
-    } else if (item.isFile() && item.name.endsWith('.mdx')) {
-      // Key: relative path without .mdx, with forward slashes
-      let rel = path.relative(baseDir, fullPath).replace(/\\/g, '/');
-      if (rel.endsWith('.mdx')) rel = rel.slice(0, -4);
-      files[rel] = fullPath;
-    }
-  }
-
-  return files;
-}
-
-// Function to clean MDX content for LLM consumption
+/** Cleans MDX content for LLM consumption (keeps code-fence languages). */
 function cleanMdxContent(content: string): string {
-  // Remove JSX components but keep their content
-  let cleaned = content
+  return content
     // Remove JSX opening/closing tags but keep content
     .replace(/<(\w+)([^>]*?)>/g, '')
     .replace(/<\/\w+>/g, '')
-    // Remove code block language specifiers
-    .replace(/```(\w+)/g, '```')
     // Remove HTML comments
     .replace(/<!--[\s\S]*?-->/g, '')
     // Remove excessive whitespace
     .replace(/\n\s*\n\s*\n/g, '\n\n')
-    // Remove leading/trailing whitespace
     .trim();
+}
 
-  return cleaned;
+/** Resolves a navigation page key to its MDX file key, or null when missing. */
+function resolveFileKey(navKey: string, mdxFilesMap: Record<string, string>): string | null {
+  if (mdxFilesMap[navKey]) return navKey;
+  if (mdxFilesMap[`${navKey}/index`]) return `${navKey}/index`;
+  return null;
 }
 
-// Function to generate LLM text content in dory.json order, then append any extra files
-function generateLlmContent(docsDir: string): string {
+/**
+ * Generates a spec-shaped llms.txt (https://llmstxt.org): H1 site name,
+ * blockquote summary, then per-tab sections with link lists to the .mdx
+ * source files.
+ */
+export function generateLlmsIndex(docsDir: string): string {
+  const config = readDoryConfig(docsDir) || {};
+  const siteName: string = config.name || 'Documentation';
+  const baseUrl: string = config.url || '';
+  const mdxFilesMap = findMdxFilesMap(docsDir, docsDir);
+
+  let output = `# ${siteName}\n`;
+  if (config.description) {
+    output += `\n> ${config.description}\n`;
+  }
+
+  const tabs: any[] = config.navigation?.tabs || [];
+  for (const tab of tabs) {
+    const links: string[] = [];
+    const walkPages = (pages: any[]) => {
+      for (const page of pages) {
+        if (typeof page === 'string') {
+          const fileKey = resolveFileKey(page, mdxFilesMap);
+          if (!fileKey) continue;
+          const raw = fs.readFileSync(mdxFilesMap[fileKey], 'utf-8');
+          const { data } = parseFrontmatter(raw);
+          const routePath = pathFromFilename(mdxFilesMap[fileKey], docsDir);
+          const url = `${baseUrl}${routePath}.mdx`;
+          const title = data.title || page;
+          links.push(data.description ? `- [${title}](${url}): ${data.description}` : `- [${title}](${url})`);
+        } else if (typeof page === 'object' && page.pages) {
+          walkPages(page.pages);
+        }
+      }
+    };
+    for (const group of tab.groups || []) {
+      if (Array.isArray(group.pages)) walkPages(group.pages);
+    }
+    if (links.length > 0) {
+      output += `\n## ${tab.tab || 'Documentation'}\n\n${links.join('\n')}\n`;
+    }
+  }
+
+  return output.trim() + '\n';
+}
+
+/**
+ * Generates the full concatenated documentation content (llms-full.txt),
+ * in dory.json navigation order with extra files appended.
+ */
+export function generateLlmContent(docsDir: string): string {
   const navOrder = getNavigationOrder(docsDir);
   const mdxFilesMap = findMdxFilesMap(docsDir, docsDir);
   const usedKeys = new Set<string>();
   let aggregatedContent = '';
 
-  // Output files in navigation order, in order as in dory.json
-  for (const navKey of navOrder) {
-    // Try both with and without "index"
-    let fileKey = navKey;
-    if (mdxFilesMap[fileKey]) {
-      usedKeys.add(fileKey);
-    } else if (mdxFilesMap[`${fileKey}/index`]) {
-      fileKey = `${fileKey}/index`;
-      usedKeys.add(fileKey);
-    } else {
-      continue;
-    }
-    const filePath = mdxFilesMap[fileKey];
-    const rawContent = fs.readFileSync(filePath, 'utf-8');
+  const appendFile = (fileKey: string) => {
+    const rawContent = fs.readFileSync(mdxFilesMap[fileKey], 'utf-8');
     const { data, content } = parseFrontmatter(rawContent);
-    const cleanedContent = cleanMdxContent(content);
 
-    // Add file header
     aggregatedContent += `\n# ${fileKey}.mdx\n`;
-
-    // Add title if available
     if (data.title) {
       aggregatedContent += `## ${data.title}\n\n`;
     }
-
-    // Add description if available
     if (data.description) {
       aggregatedContent += `${data.description}\n\n`;
     }
-
-    // Add cleaned content
-    aggregatedContent += cleanedContent + '\n\n';
+    aggregatedContent += cleanMdxContent(content) + '\n\n';
     aggregatedContent += '---\n\n';
+  };
+
+  // Output files in navigation order, in order as in dory.json
+  for (const navKey of navOrder) {
+    const fileKey = resolveFileKey(navKey, mdxFilesMap);
+    if (!fileKey) continue;
+    usedKeys.add(fileKey);
+    appendFile(fileKey);
   }
 
   // Output any extra files not in navigation order, sorted
   const extraKeys = Object.keys(mdxFilesMap).filter(k => !usedKeys.has(k)).sort();
   for (const fileKey of extraKeys) {
-    const filePath = mdxFilesMap[fileKey];
-    const rawContent = fs.readFileSync(filePath, 'utf-8');
-    const { data, content } = parseFrontmatter(rawContent);
-    const cleanedContent = cleanMdxContent(content);
-
-    aggregatedContent += `\n# ${fileKey}.mdx\n`;
-    if (data.title) {
-      aggregatedContent += `## ${data.title}\n\n`;
-    }
-    if (data.description) {
-      aggregatedContent += `${data.description}\n\n`;
-    }
-    aggregatedContent += cleanedContent + '\n\n';
-    aggregatedContent += '---\n\n';
+    appendFile(fileKey);
   }
 
-  // Remove leading newlines for the very first file, so the first file is at the very top
   return aggregatedContent.replace(/^\n+/, '').trim();
 }
 
+/**
+ * Build plugin that emits llms.txt (spec-shaped index) and llms-full.txt
+ * (full concatenated documentation).
+ */
 export function llmTxtGenerator(options: LlmTxtGeneratorOptions = {}): Plugin {
   return {
     name: 'llm-txt-generator',
     generateBundle() {
       const docsDir = options.docsDir || path.join(process.cwd(), 'docs');
-      const aggregatedContent = generateLlmContent(docsDir);
+      const indexContent = generateLlmsIndex(docsDir);
+      const fullContent = generateLlmContent(docsDir);
 
       this.emitFile({
         type: 'asset',
         fileName: 'llms.txt',
-        source: aggregatedContent
+        source: indexContent
       });
 
-      // For logging, count the number of MDX files processed
-      const mdxFilesMap = findMdxFilesMap(docsDir, docsDir);
-      const mdxFilesCount = Object.keys(mdxFilesMap).length;
+      this.emitFile({
+        type: 'asset',
+        fileName: 'llms-full.txt',
+        source: fullContent
+      });
 
-      console.log(`✅ Generated LLM text file`);
+      const mdxFilesCount = Object.keys(findMdxFilesMap(docsDir, docsDir)).length;
+      console.log(`✅ Generated llms.txt and llms-full.txt`);
       console.log(`📄 Processed ${mdxFilesCount} MDX files`);
-      console.log(`📝 Total content size: ${aggregatedContent.length} characters`);
+      console.log(`📝 Full content size: ${fullContent.length} characters`);
     }
   };
-}
+}

package/src/plugins/prerender.ts

@@ -1,11 +1,20 @@
 import type { Plugin, ResolvedConfig } from 'vite';
 import { readFileSync, writeFileSync, mkdirSync, existsSync } from 'fs';
 import { resolve } from 'path';
+import {
+  readDoryConfig,
+  findMdxFilesMap,
+  pathFromFilename,
+  parseFrontmatter,
+  stripMdxToText,
+  escapeHtml,
+  applyTitleTemplate,
+} from './shared';
 
 /**
  * Generates per-route HTML files with page-specific meta tags and JSON-LD
  * at build time. The SSR step (render-routes.mjs) later injects rendered
- * content into these files.
+ * content into these files. Also emits a prerendered 404.html.
  */
 export function prerender(): Plugin {
   let config: ResolvedConfig;
@@ -30,83 +39,140 @@ export function prerender(): Plugin {
         const frontmatter: Array<Record<string, any>> = JSON.parse(readFileSync(frontmatterPath, 'utf-8'));
         const baseHtml = readFileSync(indexHtmlPath, 'utf-8');
 
-        // Read dory.json for site-level config
-        const doryJsonPath = resolve(process.cwd(), 'docs', 'dory.json');
-        let siteConfig: Record<string, any> = {};
-        if (existsSync(doryJsonPath)) {
-          siteConfig = JSON.parse(readFileSync(doryJsonPath, 'utf-8'));
+        const siteConfig = readDoryConfig() || {};
+        const seo = siteConfig.seo || {};
+        const baseUrl: string = siteConfig.url || '';
+        const siteName: string = siteConfig.name || 'Documentation';
+
+        if (!baseUrl) {
+          console.warn(
+            '⚠️  prerender: `url` is not set in dory.json — canonical and og:url tags will be omitted from all pages. Set `url` for proper SEO.'
+          );
         }
 
-        const baseUrl = siteConfig.url || '';
-        const siteName = siteConfig.name || 'Documentation';
+        // Resolve seo.ogImage to an absolute URL when possible
+        const rawOgImage: string = seo.ogImage || '';
+        const ogImage = rawOgImage && baseUrl && !rawOgImage.startsWith('http')
+          ? `${baseUrl}${rawOgImage.startsWith('/') ? '' : '/'}${rawOgImage}`
+          : rawOgImage;
+        const hasImage = Boolean(ogImage || siteConfig.image);
+
+        // Map route paths to MDX source files for description fallbacks
+        const docsDir = resolve(process.cwd(), 'docs');
+        const mdxFilesMap = existsSync(docsDir) ? findMdxFilesMap(docsDir, docsDir) : {};
+        const fileByRoute: Record<string, string> = {};
+        for (const filePath of Object.values(mdxFilesMap)) {
+          fileByRoute[pathFromFilename(filePath, docsDir)] = filePath;
+        }
+
+        const missingDescriptions: string[] = [];
         let count = 0;
 
         for (const page of frontmatter) {
           const routePath = page.path;
           if (!routePath || routePath === '/') continue;
 
-          const title = page.title || siteName;
-          const description = page.description || `${siteName} - ${title}`;
-          const fullUrl = `${baseUrl}${routePath}`;
+          const title = applyTitleTemplate(page.title || siteName, seo.titleTemplate, siteName);
+          const description = page.description || fallbackDescription(fileByRoute[routePath]);
+          if (!page.description) {
+            missingDescriptions.push(routePath);
+          }
+          const fullUrl = baseUrl ? `${baseUrl}${routePath}` : '';
 
           let html = baseHtml;
+          // Function replacements so `$&` etc. in values cannot corrupt output
+          const setTag = (re: RegExp, replacement: string) => {
+            html = html.replace(re, () => replacement);
+          };
+          const injectHeadTag = (tag: string) => {
+            html = html.replace('</head>', () => `  ${tag}\n  </head>`);
+          };
 
-          // Replace title
-          html = html.replace(/<title>.*?<\/title>/, `<title>${escapeHtml(title)}</title>`);
-
-          // Replace meta description
-          html = html.replace(
+          setTag(/<title>.*?<\/title>/, `<title>${escapeHtml(title)}</title>`);
+          setTag(
             /<meta name="description" content=".*?" \/>/,
-            `<meta name="description" content="${escapeAttr(description)}" />`
+            `<meta name="description" content="${escapeHtml(description)}" />`
           );
 
-          // Replace OG tags
-          html = html.replace(
+          // Open Graph tags
+          setTag(
             /<meta property="og:title" content=".*?" \/>/,
-            `<meta property="og:title" content="${escapeAttr(title)}" />`
+            `<meta property="og:title" content="${escapeHtml(title)}" />`
           );
-          html = html.replace(
+          setTag(
             /<meta property="og:description" content=".*?" \/>/,
-            `<meta property="og:description" content="${escapeAttr(description)}" />`
+            `<meta property="og:description" content="${escapeHtml(description)}" />`
           );
-          html = html.replace(
-            /<meta property="og:url" content=".*?" \/>/,
-            `<meta property="og:url" content="${escapeAttr(fullUrl)}" />`
+          setTag(
+            /<meta property="og:type" content=".*?" \/>/,
+            '<meta property="og:type" content="article" />'
           );
+          if (ogImage) {
+            setTag(
+              /<meta property="og:image" content=".*?" \/>/,
+              `<meta property="og:image" content="${escapeHtml(ogImage)}" />`
+            );
+            setTag(
+              /<meta name="twitter:image" content=".*?" \/>/,
+              `<meta name="twitter:image" content="${escapeHtml(ogImage)}" />`
+            );
+          }
+          if (!hasImage) {
+            // No shareable image — a large-image card would render broken
+            setTag(
+              /<meta name="twitter:card" content=".*?" \/>/,
+              '<meta name="twitter:card" content="summary" />'
+            );
+          }
 
-          // Replace Twitter tags
-          html = html.replace(
+          // Twitter tags
+          setTag(
             /<meta name="twitter:title" content=".*?" \/>/,
-            `<meta name="twitter:title" content="${escapeAttr(title)}" />`
+            `<meta name="twitter:title" content="${escapeHtml(title)}" />`
           );
-          html = html.replace(
+          setTag(
             /<meta name="twitter:description" content=".*?" \/>/,
-            `<meta name="twitter:description" content="${escapeAttr(description)}" />`
+            `<meta name="twitter:description" content="${escapeHtml(description)}" />`
           );
 
-          // Update canonical link (always replace, never skip)
-          html = html.replace(
-            /<link rel="canonical" href=".*?" \/>/,
-            `<link rel="canonical" href="${escapeAttr(fullUrl)}" />`
-          );
+          // Canonical + og:url — only when config.url is set. When it is not,
+          // the metadata injector already removed these tags from the base HTML.
+          if (fullUrl) {
+            setTag(
+              /<meta property="og:url" content=".*?" \/>/,
+              `<meta property="og:url" content="${escapeHtml(fullUrl)}" />`
+            );
+            setTag(
+              /<link rel="canonical" href=".*?" \/>/,
+              `<link rel="canonical" href="${escapeHtml(fullUrl)}" />`
+            );
+          }
 
-          // Inject per-page robots directive if specified in frontmatter
+          // Per-page robots directive from frontmatter
           if (page.robots) {
-            html = html.replace(
-              '</head>',
-              `  <meta name="robots" content="${escapeAttr(page.robots)}" />\n  </head>`
-            );
+            injectHeadTag(`<meta name="robots" content="${escapeHtml(String(page.robots))}" />`);
           }
 
-          // Inject JSON-LD structured data
+          // JSON-LD structured data
+          const dateModified = toIsoDate(page.lastUpdated);
           const jsonLd = JSON.stringify({
             '@context': 'https://schema.org',
             '@graph': [
+              {
+                '@type': 'WebSite',
+                'name': siteName,
+                ...(baseUrl ? { '@id': `${baseUrl}/#website`, 'url': baseUrl } : {}),
+              },
               {
                 '@type': 'Article',
                 'headline': title,
                 'description': description,
                 ...(fullUrl ? { 'url': fullUrl } : {}),
+                ...(dateModified ? { 'dateModified': dateModified } : {}),
+                'publisher': { '@type': 'Organization', 'name': siteName },
+                'isPartOf': baseUrl
+                  ? { '@id': `${baseUrl}/#website` }
+                  : { '@type': 'WebSite', 'name': siteName },
               },
               {
                 '@type': 'BreadcrumbList',
@@ -126,12 +192,9 @@ export function prerender(): Plugin {
                 ]
               }
             ]
-          });
+          }).replace(/</g, '\\u003c');
 
-          html = html.replace(
-            '</head>',
-            `  <script type="application/ld+json">${jsonLd}</script>\n  </head>`
-          );
+          injectHeadTag(`<script type="application/ld+json">${jsonLd}</script>`);
 
           // Write to route directory
           const routeDir = resolve(outDir, routePath.replace(/^\//, ''));
@@ -140,6 +203,14 @@ export function prerender(): Plugin {
           count++;
         }
 
+        if (missingDescriptions.length > 0) {
+          console.warn(
+            `⚠️  prerender: ${missingDescriptions.length} page(s) missing a frontmatter description (using page text fallback): ${missingDescriptions.join(', ')}`
+          );
+        }
+
+        write404(outDir, baseHtml, siteName, frontmatter);
+
         console.log(`✅ Prerendered ${count} routes with page-specific meta tags`);
       } catch (error) {
         console.error('prerender failed:', error);
@@ -148,10 +219,45 @@ export function prerender(): Plugin {
   };
 }
 
-function escapeHtml(str: string): string {
-  return str.replace(/&/g, '&amp;').replace(/</g, '&lt;').replace(/>/g, '&gt;');
+/** Builds a ~155 char description from the page's stripped MDX text. */
+function fallbackDescription(mdxFilePath: string | undefined): string {
+  if (!mdxFilePath || !existsSync(mdxFilePath)) return '';
+  const { content } = parseFrontmatter(readFileSync(mdxFilePath, 'utf-8'));
+  const text = stripMdxToText(content).replace(/\s+/g, ' ').trim();
+  if (text.length <= 155) return text;
+  return `${text.slice(0, 155).replace(/\s+\S*$/, '')}…`;
 }
 
-function escapeAttr(str: string): string {
-  return str.replace(/&/g, '&amp;').replace(/"/g, '&quot;').replace(/</g, '&lt;').replace(/>/g, '&gt;');
+/** Normalizes a frontmatter lastUpdated value (string or Date) to an ISO date. */
+function toIsoDate(value: unknown): string {
+  if (!value) return '';
+  const date = new Date(value as string | number | Date);
+  if (isNaN(date.getTime())) return '';
+  return date.toISOString().split('T')[0];
+}
+
+/** Emits a prerendered 404.html with noindex and a link to the first nav page. */
+function write404(
+  outDir: string,
+  baseHtml: string,
+  siteName: string,
+  frontmatter: Array<Record<string, any>>
+): void {
+  const firstPage = frontmatter.find(page => page.path && page.path !== '/');
+  const firstHref: string = firstPage?.path || '/';
+  const firstTitle: string = firstPage?.title || siteName;
+
+  let html = baseHtml;
+  html = html.replace(/<title>.*?<\/title>/, () => `<title>Page not found — ${escapeHtml(siteName)}</title>`);
+  html = html.replace('</head>', () => '  <meta name="robots" content="noindex" />\n  </head>');
+  html = html.replace(
+    '<div id="app"></div>',
+    () =>
+      '<div id="app"><main style="max-width:36rem;margin:6rem auto;padding:0 1.5rem;font-family:sans-serif;text-align:center">' +
+      '<h1>Page not found</h1>' +
+      '<p>The page you are looking for does not exist.</p>' +
+      `<p><a href="${escapeHtml(firstHref)}">Go to ${escapeHtml(firstTitle)}</a></p>` +
+      '</main></div>'
+  );
+  writeFileSync(resolve(outDir, '404.html'), html, 'utf-8');
 }

package/src/plugins/robots-generator.ts

@@ -1,34 +1,33 @@
 import type { Plugin } from 'vite';
-import { readFileSync, existsSync } from 'fs';
-import { resolve } from 'path';
+import { readDoryConfig } from './shared';
 
 /**
- * Generates robots.txt with a sitemap reference at build time.
+ * Generates robots.txt at build time. Emits `Disallow: /` when seo.noindex
+ * is set, appends any seo.robotsTxt lines, and references the sitemap when
+ * `url` is configured.
  */
 export function robotsGenerator(): Plugin {
   return {
     name: 'robots-generator',
     generateBundle() {
-      const doryJsonPath = resolve(process.cwd(), 'docs', 'dory.json');
+      const config = readDoryConfig() || {};
+      const seo = config.seo || {};
 
-      let sitemapLine = '';
-      if (existsSync(doryJsonPath)) {
-        try {
-          const config = JSON.parse(readFileSync(doryJsonPath, 'utf-8'));
-          if (config.url) {
-            sitemapLine = `\nSitemap: ${config.url}/sitemap.xml\n`;
-          }
-        } catch {
-          // Non-critical, continue without sitemap reference
-        }
+      const lines = ['User-agent: *'];
+      lines.push(seo.noindex ? 'Disallow: /' : 'Allow: /');
+
+      if (Array.isArray(seo.robotsTxt) && seo.robotsTxt.length > 0) {
+        lines.push('', ...seo.robotsTxt);
       }
 
-      const robots = `User-agent: *\nAllow: /\n${sitemapLine}`;
+      if (config.url && !seo.noindex) {
+        lines.push('', `Sitemap: ${config.url}/sitemap.xml`);
+      }
 
       this.emitFile({
         type: 'asset',
         fileName: 'robots.txt',
-        source: robots
+        source: lines.join('\n') + '\n'
       });
 
       console.log('✅ Generated robots.txt');