@clidey/dory 0.47.0 → 0.49.0

package/src/plugins/sanitize.test.ts

@@ -0,0 +1,104 @@
+import { describe, it, expect } from 'vitest';
+import { preprocessMdxTags, isInsideCodeBlock, isInsideCodeSpan } from './sanitize';
+
+/** Run the preprocessor over MDX input and return the transformed code. */
+function run(input: string): string {
+  const result = preprocessMdxTags().transform(input, 'test.mdx');
+  return (result as { code: string }).code;
+}
+
+describe('isInsideCodeBlock', () => {
+  it('detects content between line-start fences', () => {
+    const code = '```\nconst x = 1;\n```\nafter';
+    expect(isInsideCodeBlock(code, code.indexOf('const'))).toBe(true);
+    expect(isInsideCodeBlock(code, code.indexOf('after'))).toBe(false);
+  });
+
+  it('ignores inline triple backticks (not at line start)', () => {
+    const code = 'Use ``` to fence code blocks.\nplain text here';
+    expect(isInsideCodeBlock(code, code.indexOf('plain'))).toBe(false);
+  });
+
+  it('handles indented fences and CRLF line endings', () => {
+    const code = 'a\r\n  ```js\r\ninside\r\n  ```\r\nb';
+    expect(isInsideCodeBlock(code, code.indexOf('inside'))).toBe(true);
+    expect(isInsideCodeBlock(code, code.indexOf('b'))).toBe(false);
+  });
+});
+
+describe('isInsideCodeSpan', () => {
+  it('detects content inside a single-backtick span', () => {
+    const code = 'a `<div>` b';
+    expect(isInsideCodeSpan(code, code.indexOf('<div>'))).toBe(true);
+    expect(isInsideCodeSpan(code, code.indexOf('b'))).toBe(false);
+  });
+
+  it('pairs equal-length backtick runs (double-backtick span with inner backtick)', () => {
+    const code = 'A ``b ` c`` and <Unknown> tag';
+    expect(isInsideCodeSpan(code, code.indexOf('b `'))).toBe(true);
+    // The single backtick inside the double-backtick span must not
+    // open a new span that swallows the rest of the line.
+    expect(isInsideCodeSpan(code, code.indexOf('<Unknown>'))).toBe(false);
+  });
+
+  it('treats an unmatched backtick run as literal text', () => {
+    const code = 'odd ` backtick <Foo> here';
+    expect(isInsideCodeSpan(code, code.indexOf('<Foo>'))).toBe(false);
+  });
+});
+
+describe('preprocessMdxTags', () => {
+  it('returns undefined for non-mdx files', () => {
+    expect(preprocessMdxTags().transform('<Foo>', 'file.ts')).toBeUndefined();
+  });
+
+  it('escapes bare unknown pseudo-tags in backticks', () => {
+    expect(run('This <UnknownTag> is not JSX')).toBe('This `<UnknownTag>` is not JSX');
+  });
+
+  it('preserves known components with matched closing tags', () => {
+    expect(run('<Note>hello</Note>')).toBe('<Note>hello</Note>');
+  });
+
+  it('escapes known components without a closing tag', () => {
+    expect(run('<div>no closing tag')).toBe('`<div>`no closing tag');
+  });
+
+  it('preserves nested same-name tags without leaking an orphan closer', () => {
+    const input = '<div><div>inner</div></div>';
+    expect(run(input)).toBe(input);
+  });
+
+  it('leaves tags inside fenced code blocks untouched', () => {
+    const input = '```\n<UnknownTag>\n<div>\n```\n';
+    expect(run(input)).toBe(input);
+  });
+
+  it('handles CRLF input with fenced code blocks', () => {
+    const input = 'text\r\n```\r\n<UnknownTag>\r\n```\r\n';
+    expect(run(input)).toBe(input);
+  });
+
+  it('does not corrupt double-backtick code spans', () => {
+    const input = 'A ``b `<div>` c`` d';
+    expect(run(input)).toBe(input);
+  });
+
+  it('still escapes tags after a double-backtick span (no parity confusion)', () => {
+    expect(run('A ``b ` c`` and <Unknown> tag')).toBe('A ``b ` c`` and `<Unknown>` tag');
+  });
+
+  it('does not treat inline triple backticks as a code fence', () => {
+    expect(run('Use ``` to fence.\n\n<Unknown>')).toBe('Use ``` to fence.\n\n`<Unknown>`');
+  });
+
+  it('escapes unknown self-closing tags and preserves known ones', () => {
+    expect(run('<VideoPlayer src="x" />')).toBe('`<VideoPlayer src="x" />`');
+    expect(run('line<br/>break')).toBe('line<br/>break');
+  });
+
+  it('leaves tags inside single-backtick inline code untouched', () => {
+    const input = 'use `<UnknownTag>` in your doc';
+    expect(run(input)).toBe(input);
+  });
+});

package/src/plugins/sanitize.ts

@@ -17,7 +17,7 @@ export const remarkSafeVars = () => {
 const KNOWN_COMPONENTS = [
   // Text components
   'h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'p', 'Row', 'Col', 'a', 'img',
-  // Code components  
+  // Code components
   'code', 'CodeGroup', 'pre',
   // UI components
   'Accordion', 'AccordionGroup', 'Icon', 'Card', 'CardGroup',
@@ -33,43 +33,85 @@ const KNOWN_COMPONENTS = [
   'address', 'cite', 'q', 'dfn'
 ];
 
-// Helper: returns true if the given code string is inside a code block (```)
-function isInsideCodeBlock(code: string, idx: number): boolean {
-  // Count the number of ``` before idx
-  const before = code.slice(0, idx);
-  const codeBlockMatches = before.match(/```/g);
-  // Odd number of ``` means inside a code block
-  return codeBlockMatches ? codeBlockMatches.length % 2 === 1 : false;
+/**
+ * Returns true if the character at `idx` is inside a fenced code block.
+ * Fences only count when they start a line (optionally indented) — inline
+ * triple backticks in prose do not open or close a block. Offsets on a
+ * fence line itself are treated as inside, so fence lines are never modified.
+ */
+export function isInsideCodeBlock(code: string, idx: number): boolean {
+  let inFence = false;
+  let pos = 0;
+  for (const line of code.split('\n')) {
+    const lineEnd = pos + line.length;
+    const isFenceLine = /^\s*```/.test(line);
+    if (idx >= pos && idx <= lineEnd) {
+      return inFence || isFenceLine;
+    }
+    if (isFenceLine) {
+      inFence = !inFence;
+    }
+    pos = lineEnd + 1;
+  }
+  return inFence;
 }
 
-// Helper: checks if a known component tag has a matching closing tag
-function hasMatchingClosingTag(code: string, tagName: string, openingTagOffset: number): boolean {
-  // Simple approach: look for the next occurrence of either an opening or closing tag of the same type
-  const afterOpening = code.slice(openingTagOffset + `<${tagName}>`.length);
-  
-  // Find the first occurrence of either opening or closing tag
-  const nextOpeningMatch = afterOpening.match(new RegExp(`<${tagName}>`, 'i'));
-  const nextClosingMatch = afterOpening.match(new RegExp(`</${tagName}>`, 'i'));
-  
-  // If no closing tag found at all
-  if (!nextClosingMatch) {
-    return false;
+/**
+ * Returns true if the character at `idx` is inside an inline code span.
+ * Follows CommonMark pairing: a run of N backticks opens a span closed by
+ * the next run of exactly N backticks; unmatched runs are literal text.
+ * Backtick runs inside fenced code blocks are ignored.
+ */
+export function isInsideCodeSpan(code: string, idx: number): boolean {
+  const runs: { index: number; length: number }[] = [];
+  const re = /`+/g;
+  let m: RegExpExecArray | null;
+  while ((m = re.exec(code)) !== null) {
+    if (!isInsideCodeBlock(code, m.index)) {
+      runs.push({ index: m.index, length: m[0].length });
+    }
   }
-  
-  const nextOpeningIndex = nextOpeningMatch ? nextOpeningMatch.index! : Infinity;
-  const nextClosingIndex = nextClosingMatch ? nextClosingMatch.index! : Infinity;
-  
-  // If the closing tag comes before any other opening tag, this tag is properly closed
-  if (nextClosingIndex < nextOpeningIndex) {
-    // Check if the closing tag is inside a code block
-    const closingTagOffset = openingTagOffset + `<${tagName}>`.length + nextClosingIndex;
-    if (isInsideCodeBlock(code, closingTagOffset)) {
-      return false;
+
+  let i = 0;
+  while (i < runs.length) {
+    const open = runs[i];
+    if (open.index >= idx) return false;
+
+    // Find the matching closer: the next run of the same length
+    let j = i + 1;
+    while (j < runs.length && runs[j].length !== open.length) j++;
+
+    if (j < runs.length) {
+      if (idx > open.index && idx < runs[j].index) return true;
+      i = j + 1;
+    } else {
+      // Unmatched run is literal text, not a span opener
+      i++;
+    }
+  }
+  return false;
+}
+
+// Helper: checks if a known component tag has a matching closing tag,
+// tracking nesting depth so nested same-name tags pair up correctly.
+function hasMatchingClosingTag(code: string, tagName: string, openingTagOffset: number): boolean {
+  const re = new RegExp(`</?${tagName}>`, 'gi');
+  re.lastIndex = openingTagOffset + `<${tagName}>`.length;
+  let depth = 1;
+  let m: RegExpExecArray | null;
+  while ((m = re.exec(code)) !== null) {
+    // Tags inside code blocks or inline code spans are literal text
+    if (isInsideCodeBlock(code, m.index) || isInsideCodeSpan(code, m.index)) {
+      continue;
+    }
+    if (m[0].startsWith('</')) {
+      depth--;
+      if (depth === 0) return true;
+    } else {
+      depth++;
     }
-    return true;
   }
-  
-  return false; // Another opening tag comes before the closing tag, so this one is not properly closed
+  return false;
 }
 
 // Allow any <[a-z][a-z0-9]*> tag inside code blocks to be left as-is (not parsed as JSX).
@@ -80,16 +122,18 @@ export function preprocessMdxTags() {
     transform(code: string, id: string) {
       if (!id.endsWith('.mdx')) return;
 
-      // Remove leading whitespace from code fence blocks
-      let processed = code.replace(/^(\s+)(```\w*)/gm, '$2');
+      // Remove leading whitespace from code fence blocks.
+      // Only horizontal whitespace: \s would also swallow the \n of CRLF
+      // line endings (multiline ^ matches after \r in JS).
+      let processed = code.replace(/^([ \t]+)(```\w*)/gm, '$2');
 
       // Handle self-closing tags first (like <VideoPlayer />)
       processed = processed.replace(/<([A-Za-z][A-Za-z0-9]*)[^>]*\/>/g, (match, tag, offset) => {
-        // If inside a code block, do not touch
-        if (isInsideCodeBlock(processed, offset)) {
+        // If inside a code block or inline code span, do not touch
+        if (isInsideCodeBlock(processed, offset) || isInsideCodeSpan(processed, offset)) {
           return match;
         }
-        
+
         // If unknown component, treat as text
         if (!KNOWN_COMPONENTS.includes(tag)) {
           return `\`${match}\``;
@@ -100,11 +144,11 @@ export function preprocessMdxTags() {
       // Only replace unrecognized <someTag> with `"<someTag>"` if NOT inside a code block
       // This regex finds all <tag> occurrences (fixed to handle uppercase)
       processed = processed.replace(/<([A-Za-z][A-Za-z0-9]*)>/g, (match, tag, offset) => {
-        // If inside a code block, do not touch
-        if (isInsideCodeBlock(processed, offset)) {
+        // If inside a code block or inline code span, do not touch
+        if (isInsideCodeBlock(processed, offset) || isInsideCodeSpan(processed, offset)) {
           return match;
         }
-        
+
         // If known component, check if it has a closing tag
         if (KNOWN_COMPONENTS.includes(tag)) {
           // If it doesn't have a matching closing tag, treat it as text
@@ -115,23 +159,17 @@ export function preprocessMdxTags() {
           return match;
         }
         // For unknown components, always treat as text (wrap in backticks)
-        // If already inside backticks, do not wrap again
-        const before = processed.slice(0, offset);
-        const backtickMatches = before.match(/`+/g);
-        const insideBackticks = backtickMatches ? backtickMatches.reduce((acc, s) => acc + s.length, 0) % 2 === 1 : false;
-        if (insideBackticks) {
-          return match;
-        }
         return `\`<${tag}>\``;
       });
 
       // This regex finds all <tag> and </tag> occurrences (fixed to handle uppercase)
       processed = processed.replace(/<\/?([A-Za-z][A-Za-z0-9]*)>/g, (match, tag, offset) => {
-        // If inside a code block, do not touch
-        if (isInsideCodeBlock(processed, offset)) {
+        // If inside a code block or inline code span (including spans added
+        // by the previous pass), do not touch
+        if (isInsideCodeBlock(processed, offset) || isInsideCodeSpan(processed, offset)) {
           return match;
         }
-        
+
         // If known component, check if it has a closing tag (only for opening tags)
         if (KNOWN_COMPONENTS.includes(tag)) {
           // Only check for matching closing tag for opening tags
@@ -144,13 +182,6 @@ export function preprocessMdxTags() {
           return match;
         }
         // For unknown components, always treat as text (wrap in backticks)
-        // If already inside backticks, do not wrap again
-        const before = processed.slice(0, offset);
-        const backtickMatches = before.match(/`+/g);
-        const insideBackticks = backtickMatches ? backtickMatches.reduce((acc, s) => acc + s.length, 0) % 2 === 1 : false;
-        if (insideBackticks) {
-          return match;
-        }
         return `\`${match}\``;
       });
 
@@ -160,4 +191,4 @@ export function preprocessMdxTags() {
       };
     },
   };
-}
+}

package/src/plugins/shared.ts

@@ -0,0 +1,139 @@
+import matter from 'gray-matter';
+import fs from 'fs';
+import path from 'path';
+
+/**
+ * Shared helpers for build plugins that read docs/*.mdx and docs/dory.json.
+ * Consolidated from near-identical copies in frontmatter-generator,
+ * frontmatter-dev-server, llm-txt-generator, and llm-txt-dev-server.
+ */
+
+/**
+ * Parses YAML frontmatter from MDX content using gray-matter.
+ * Handles CRLF line endings and real YAML (nested values, dates, quotes).
+ */
+export function parseFrontmatter(content: string): { data: Record<string, any>; content: string } {
+  const parsed = matter(content);
+  return { data: parsed.data, content: parsed.content };
+}
+
+/**
+ * Reads and parses docs/dory.json. Returns null when the file is missing
+ * or unparseable (a parse error is logged).
+ */
+export function readDoryConfig(docsDir?: string): Record<string, any> | null {
+  const doryJsonPath = path.join(docsDir || path.join(process.cwd(), 'docs'), 'dory.json');
+  if (!fs.existsSync(doryJsonPath)) return null;
+  try {
+    return JSON.parse(fs.readFileSync(doryJsonPath, 'utf-8'));
+  } catch (error) {
+    const detail = error instanceof SyntaxError ? error.message : String(error);
+    console.error(`Failed to parse ${doryJsonPath}: ${detail}`);
+    return null;
+  }
+}
+
+/**
+ * Reads dory.json navigation and returns the flat, ordered list of page keys.
+ */
+export function getNavigationOrder(docsDir: string): string[] {
+  const doryJson = readDoryConfig(docsDir);
+  if (!doryJson) return [];
+  const order: string[] = [];
+
+  function walkPages(pages: any[]) {
+    for (const page of pages) {
+      if (typeof page === 'string') {
+        order.push(page);
+      } else if (typeof page === 'object' && page.pages) {
+        walkPages(page.pages);
+      }
+    }
+  }
+
+  if (doryJson.navigation && Array.isArray(doryJson.navigation.tabs)) {
+    for (const tab of doryJson.navigation.tabs) {
+      if (tab.groups && Array.isArray(tab.groups)) {
+        for (const group of tab.groups) {
+          if (group.pages && Array.isArray(group.pages)) {
+            walkPages(group.pages);
+          }
+        }
+      }
+    }
+  }
+  return order;
+}
+
+/**
+ * Converts an MDX filename to the route path used by the store,
+ * e.g. docs/getting-started/installation.mdx -> /getting-started/installation.
+ */
+export function pathFromFilename(filename: string, docsDir: string): string {
+  return filename
+    .replace(docsDir, '')
+    .replace(/\/?index\.mdx$/, '/')
+    .replace(/\.mdx$/, '')
+    .replace(/\\/g, '/')
+    .toLowerCase();
+}
+
+/**
+ * Recursively finds all MDX files under a directory, keyed by relative
+ * path without the .mdx extension (forward slashes).
+ */
+export function findMdxFilesMap(dir: string, baseDir: string): Record<string, string> {
+  const files: Record<string, string> = {};
+  const items = fs.readdirSync(dir, { withFileTypes: true });
+
+  for (const item of items) {
+    const fullPath = path.join(dir, item.name);
+    if (item.isDirectory()) {
+      Object.assign(files, findMdxFilesMap(fullPath, baseDir));
+    } else if (item.isFile() && item.name.endsWith('.mdx')) {
+      let rel = path.relative(baseDir, fullPath).replace(/\\/g, '/');
+      if (rel.endsWith('.mdx')) rel = rel.slice(0, -4);
+      files[rel] = fullPath;
+    }
+  }
+
+  return files;
+}
+
+/**
+ * Strips MDX/JSX syntax to extract plain text (for search indexing and
+ * meta description fallbacks).
+ */
+export function stripMdxToText(content: string): string {
+  return content
+    .replace(/^import\s+.*$/gm, '')           // import statements
+    .replace(/^export\s+.*$/gm, '')           // export statements
+    .replace(/```[\s\S]*?```/g, '')           // fenced code blocks
+    .replace(/<[^>]+>/g, '')                  // JSX/HTML tags
+    .replace(/\[([^\]]*)\]\([^)]*\)/g, '$1')  // markdown links -> text
+    .replace(/[#*_~`>|]/g, '')                // markdown syntax chars
+    .replace(/\n{3,}/g, '\n\n')               // collapse blank lines
+    .trim();
+}
+
+/**
+ * Escapes a string for safe use in HTML/XML text and attribute values.
+ */
+export function escapeHtml(str: string): string {
+  return str
+    .replace(/&/g, '&amp;')
+    .replace(/</g, '&lt;')
+    .replace(/>/g, '&gt;')
+    .replace(/"/g, '&quot;')
+    .replace(/'/g, '&#39;');
+}
+
+/**
+ * Applies an seo.titleTemplate (e.g. "%s — Site") to a page title.
+ * Returns the title unchanged when no template is set or the title is
+ * already the bare site title.
+ */
+export function applyTitleTemplate(title: string, template: string | undefined, siteName: string): string {
+  if (!template || title === siteName) return title;
+  return template.replace('%s', title);
+}

package/src/plugins/sitemap-generator.ts

@@ -1,72 +1,78 @@
 import type { Plugin } from 'vite';
-import { readFileSync, existsSync, statSync } from 'fs';
+import { readFileSync, existsSync } from 'fs';
 import { resolve } from 'path';
+import {
+  readDoryConfig,
+  findMdxFilesMap,
+  pathFromFilename,
+  parseFrontmatter,
+  escapeHtml,
+} from './shared';
 
 /**
- * Generates sitemap.xml from dory.json navigation structure at build time.
- * Uses actual file modification times for lastmod and calculates priority
- * based on route depth.
+ * Generates sitemap.xml at build time from all MDX files (not just those in
+ * navigation). Uses frontmatter `lastUpdated` for lastmod when present and
+ * calculates priority based on route depth. Skipped entirely when
+ * `url` is missing or `seo.noindex` is set.
  */
 export function sitemapGenerator(): Plugin {
   return {
     name: 'sitemap-generator',
     generateBundle() {
       const docsDir = resolve(process.cwd(), 'docs');
-      const doryJsonPath = resolve(docsDir, 'dory.json');
-      if (!existsSync(doryJsonPath)) {
+      const config = readDoryConfig(docsDir);
+      if (!config) {
         console.warn('sitemap-generator: dory.json not found, skipping');
         return;
       }
 
-      try {
-        const config = JSON.parse(readFileSync(doryJsonPath, 'utf-8'));
-        const baseUrl = config.url || '';
-
-        // Extract all routes from navigation
-        const routes: string[] = [];
-        function walkPages(pages: any[]) {
-          for (const page of pages) {
-            if (typeof page === 'string') {
-              routes.push(page);
-            } else if (typeof page === 'object' && page.pages) {
-              walkPages(page.pages);
-            }
-          }
-        }
+      const baseUrl: string = config.url || '';
+      if (!baseUrl) {
+        console.warn(
+          '⚠️  sitemap-generator: `url` is not set in dory.json — sitemap.xml will NOT be generated. Set `url` to enable it.'
+        );
+        return;
+      }
+      if (config.seo?.noindex) {
+        console.warn('⚠️  sitemap-generator: seo.noindex is set — sitemap.xml will NOT be generated.');
+        return;
+      }
 
-        if (config.navigation?.tabs) {
-          for (const tab of config.navigation.tabs) {
-            if (tab.groups) {
-              for (const group of tab.groups) {
-                if (group.pages) {
-                  walkPages(group.pages);
-                }
-              }
-            }
-          }
-        }
+      try {
+        const mdxFilesMap = existsSync(docsDir) ? findMdxFilesMap(docsDir, docsDir) : {};
 
-        const fallbackDate = new Date().toISOString().split('T')[0];
         let sitemap = '<?xml version="1.0" encoding="UTF-8"?>\n';
         sitemap += '<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">\n';
 
-        for (const route of routes) {
-          // Get actual file modification time
-          const mdxPath = resolve(docsDir, `${route}.mdx`);
-          let lastmod = fallbackDate;
-          if (existsSync(mdxPath)) {
-            lastmod = statSync(mdxPath).mtime.toISOString().split('T')[0];
-          }
+        // Root URL first
+        sitemap += '  <url>\n';
+        sitemap += `    <loc>${escapeHtml(`${baseUrl}/`)}</loc>\n`;
+        sitemap += '    <priority>1.0</priority>\n';
+        sitemap += '  </url>\n';
+
+        let count = 1;
+        for (const filePath of Object.values(mdxFilesMap).sort()) {
+          const routePath = pathFromFilename(filePath, docsDir);
+          if (routePath === '/') continue;
+
+          const { data } = parseFrontmatter(readFileSync(filePath, 'utf-8'));
+          // Exclude pages marked noindex in frontmatter
+          if (typeof data.robots === 'string' && data.robots.includes('noindex')) continue;
+
+          const lastmod = toIsoDate(data.lastUpdated);
 
           // Priority based on route depth: fewer segments = higher priority
-          const depth = route.split('/').length;
+          const depth = routePath.replace(/^\//, '').split('/').length;
           const priority = Math.max(0.3, 1.0 - (depth - 1) * 0.2).toFixed(1);
 
           sitemap += '  <url>\n';
-          sitemap += `    <loc>${baseUrl}/${route}</loc>\n`;
-          sitemap += `    <lastmod>${lastmod}</lastmod>\n`;
+          sitemap += `    <loc>${escapeHtml(`${baseUrl}${routePath}`)}</loc>\n`;
+          if (lastmod) {
+            sitemap += `    <lastmod>${lastmod}</lastmod>\n`;
+          }
           sitemap += `    <priority>${priority}</priority>\n`;
           sitemap += '  </url>\n';
+          count++;
         }
 
         sitemap += '</urlset>';
@@ -77,7 +83,7 @@ export function sitemapGenerator(): Plugin {
           source: sitemap
         });
 
-        console.log(`✅ Generated sitemap.xml with ${routes.length} URLs`);
+        console.log(`✅ Generated sitemap.xml with ${count} URLs`);
       } catch (error) {
         const detail = error instanceof SyntaxError ? error.message : String(error);
         console.error(`Failed to generate sitemap: ${detail}`);
@@ -85,3 +91,11 @@ export function sitemapGenerator(): Plugin {
     }
   };
 }
+
+/** Normalizes a frontmatter lastUpdated value (string or Date) to an ISO date. */
+function toIsoDate(value: unknown): string {
+  if (!value) return '';
+  const date = new Date(value as string | number | Date);
+  if (isNaN(date.getTime())) return '';
+  return date.toISOString().split('T')[0];
+}