@clidey/dory 0.30.0 → 0.31.0

package/src/plugins/llm-txt-dev-server.ts

@@ -0,0 +1,183 @@
+import type { Plugin } from 'vite';
+import fs from 'fs';
+import path from 'path';
+
+// Read dory.json to get navigation order (flat list, in order)
+function getNavigationOrder(docsDir: string): string[] {
+  const doryJsonPath = path.join(docsDir, 'dory.json');
+  if (!fs.existsSync(doryJsonPath)) return [];
+  const doryJson = JSON.parse(fs.readFileSync(doryJsonPath, 'utf-8'));
+  const order: string[] = [];
+
+  function walkPages(pages: any[]) {
+    for (const page of pages) {
+      if (typeof page === 'string') {
+        order.push(page);
+      } else if (typeof page === 'object' && page.pages) {
+        walkPages(page.pages);
+      }
+    }
+  }
+
+  if (doryJson.navigation && Array.isArray(doryJson.navigation.tabs)) {
+    for (const tab of doryJson.navigation.tabs) {
+      if (tab.groups && Array.isArray(tab.groups)) {
+        for (const group of tab.groups) {
+          if (group.pages && Array.isArray(group.pages)) {
+            walkPages(group.pages);
+          }
+        }
+      }
+    }
+  }
+  return order;
+}
+
+// Function to parse frontmatter from MDX content
+function parseFrontmatter(content: string) {
+  const frontmatterMatch = content.match(/^---\n([\s\S]*?)\n---\n/);
+  if (!frontmatterMatch) return { data: {}, content };
+
+  const frontmatter = frontmatterMatch[1];
+  const data: Record<string, string> = {};
+
+  frontmatter.split('\n').forEach(line => {
+    const [key, ...valueParts] = line.split(':');
+    if (key && valueParts.length) {
+      const value = valueParts.join(':').trim();
+      data[key.trim()] = value.replace(/^["']|["']$/g, '');
+    }
+  });
+
+  return { data, content: content.slice(frontmatterMatch[0].length) };
+}
+
+// Function to recursively find all MDX files and map by relative path (without .mdx)
+function findMdxFilesMap(dir: string, baseDir: string): Record<string, string> {
+  const files: Record<string, string> = {};
+  const items = fs.readdirSync(dir, { withFileTypes: true });
+
+  for (const item of items) {
+    const fullPath = path.join(dir, item.name);
+    if (item.isDirectory()) {
+      Object.assign(files, findMdxFilesMap(fullPath, baseDir));
+    } else if (item.isFile() && item.name.endsWith('.mdx')) {
+      // Key: relative path without .mdx, with forward slashes
+      let rel = path.relative(baseDir, fullPath).replace(/\\/g, '/');
+      if (rel.endsWith('.mdx')) rel = rel.slice(0, -4);
+      files[rel] = fullPath;
+    }
+  }
+
+  return files;
+}
+
+// Function to clean MDX content for LLM consumption
+function cleanMdxContent(content: string): string {
+  // Remove JSX components but keep their content
+  let cleaned = content
+    // Remove JSX opening/closing tags but keep content
+    .replace(/<(\w+)([^>]*?)>/g, '')
+    .replace(/<\/\w+>/g, '')
+    // Remove code block language specifiers
+    .replace(/```(\w+)/g, '```')
+    // Remove HTML comments
+    .replace(/<!--[\s\S]*?-->/g, '')
+    // Remove excessive whitespace
+    .replace(/\n\s*\n\s*\n/g, '\n\n')
+    // Remove leading/trailing whitespace
+    .trim();
+
+  return cleaned;
+}
+
+// Function to generate LLM text content in dory.json order, then append any extra files
+function generateLlmContent(docsDir: string): string {
+  const navOrder = getNavigationOrder(docsDir);
+  const mdxFilesMap = findMdxFilesMap(docsDir, docsDir);
+  const usedKeys = new Set<string>();
+  let aggregatedContent = '';
+
+  // Output files in navigation order, in order as in dory.json
+  for (const navKey of navOrder) {
+    // Try both with and without "index"
+    let fileKey = navKey;
+    if (mdxFilesMap[fileKey]) {
+      usedKeys.add(fileKey);
+    } else if (mdxFilesMap[`${fileKey}/index`]) {
+      fileKey = `${fileKey}/index`;
+      usedKeys.add(fileKey);
+    } else {
+      continue;
+    }
+    const filePath = mdxFilesMap[fileKey];
+    const rawContent = fs.readFileSync(filePath, 'utf-8');
+    const { data, content } = parseFrontmatter(rawContent);
+    const cleanedContent = cleanMdxContent(content);
+
+    // Add file header
+    aggregatedContent += `\n# ${fileKey}.mdx\n`;
+
+    // Add title if available
+    if (data.title) {
+      aggregatedContent += `## ${data.title}\n\n`;
+    }
+
+    // Add description if available
+    if (data.description) {
+      aggregatedContent += `${data.description}\n\n`;
+    }
+
+    // Add cleaned content
+    aggregatedContent += cleanedContent + '\n\n';
+    aggregatedContent += '---\n\n';
+  }
+
+  // Output any extra files not in navigation order, sorted
+  const extraKeys = Object.keys(mdxFilesMap).filter(k => !usedKeys.has(k)).sort();
+  for (const fileKey of extraKeys) {
+    const filePath = mdxFilesMap[fileKey];
+    const rawContent = fs.readFileSync(filePath, 'utf-8');
+    const { data, content } = parseFrontmatter(rawContent);
+    const cleanedContent = cleanMdxContent(content);
+
+    aggregatedContent += `\n# ${fileKey}.mdx\n`;
+    if (data.title) {
+      aggregatedContent += `## ${data.title}\n\n`;
+    }
+    if (data.description) {
+      aggregatedContent += `${data.description}\n\n`;
+    }
+    aggregatedContent += cleanedContent + '\n\n';
+    aggregatedContent += '---\n\n';
+  }
+
+  // Remove leading newlines for the very first file, so the first file is at the very top
+  return aggregatedContent.replace(/^\n+/, '').trim();
+}
+
+export function llmTxtDevServer(): Plugin {
+  return {
+    name: 'llm-txt-dev-server',
+    configureServer(server) {
+      server.middlewares.use('/llms.txt', (req, res, next) => {
+        if (req.method !== 'GET') {
+          return next();
+        }
+
+        try {
+          const docsDir = path.join(process.cwd(), 'docs');
+          const llmContent = generateLlmContent(docsDir);
+
+          res.setHeader('Content-Type', 'text/plain; charset=utf-8');
+          res.setHeader('Content-Length', Buffer.byteLength(llmContent));
+          res.end(llmContent);
+        } catch (error) {
+          console.error('Error generating LLM text:', error);
+          res.statusCode = 500;
+          res.end('Internal Server Error');
+        }
+      });
+    }
+  };
+}

package/src/plugins/llm-txt-generator.ts

@@ -0,0 +1,186 @@
+import type { Plugin } from 'vite';
+import fs from 'fs';
+import path from 'path';
+
+interface LlmTxtGeneratorOptions {
+  docsDir?: string;
+  outputPath?: string;
+}
+
+// Read dory.json to get navigation order (flat list, in order)
+function getNavigationOrder(docsDir: string): string[] {
+  const doryJsonPath = path.join(docsDir, 'dory.json');
+  if (!fs.existsSync(doryJsonPath)) return [];
+  const doryJson = JSON.parse(fs.readFileSync(doryJsonPath, 'utf-8'));
+  const order: string[] = [];
+
+  function walkPages(pages: any[]) {
+    for (const page of pages) {
+      if (typeof page === 'string') {
+        order.push(page);
+      } else if (typeof page === 'object' && page.pages) {
+        walkPages(page.pages);
+      }
+    }
+  }
+
+  if (doryJson.navigation && Array.isArray(doryJson.navigation.tabs)) {
+    for (const tab of doryJson.navigation.tabs) {
+      if (tab.groups && Array.isArray(tab.groups)) {
+        for (const group of tab.groups) {
+          if (group.pages && Array.isArray(group.pages)) {
+            walkPages(group.pages);
+          }
+        }
+      }
+    }
+  }
+  return order;
+}
+
+// Function to parse frontmatter from MDX content
+function parseFrontmatter(content: string) {
+  const frontmatterMatch = content.match(/^---\n([\s\S]*?)\n---\n/);
+  if (!frontmatterMatch) return { data: {}, content };
+
+  const frontmatter = frontmatterMatch[1];
+  const data: Record<string, string> = {};
+
+  frontmatter.split('\n').forEach(line => {
+    const [key, ...valueParts] = line.split(':');
+    if (key && valueParts.length) {
+      const value = valueParts.join(':').trim();
+      data[key.trim()] = value.replace(/^["']|["']$/g, '');
+    }
+  });
+
+  return { data, content: content.slice(frontmatterMatch[0].length) };
+}
+
+// Function to recursively find all MDX files and map by relative path (without .mdx)
+function findMdxFilesMap(dir: string, baseDir: string): Record<string, string> {
+  const files: Record<string, string> = {};
+  const items = fs.readdirSync(dir, { withFileTypes: true });
+
+  for (const item of items) {
+    const fullPath = path.join(dir, item.name);
+    if (item.isDirectory()) {
+      Object.assign(files, findMdxFilesMap(fullPath, baseDir));
+    } else if (item.isFile() && item.name.endsWith('.mdx')) {
+      // Key: relative path without .mdx, with forward slashes
+      let rel = path.relative(baseDir, fullPath).replace(/\\/g, '/');
+      if (rel.endsWith('.mdx')) rel = rel.slice(0, -4);
+      files[rel] = fullPath;
+    }
+  }
+
+  return files;
+}
+
+// Function to clean MDX content for LLM consumption
+function cleanMdxContent(content: string): string {
+  // Remove JSX components but keep their content
+  let cleaned = content
+    // Remove JSX opening/closing tags but keep content
+    .replace(/<(\w+)([^>]*?)>/g, '')
+    .replace(/<\/\w+>/g, '')
+    // Remove code block language specifiers
+    .replace(/```(\w+)/g, '```')
+    // Remove HTML comments
+    .replace(/<!--[\s\S]*?-->/g, '')
+    // Remove excessive whitespace
+    .replace(/\n\s*\n\s*\n/g, '\n\n')
+    // Remove leading/trailing whitespace
+    .trim();
+
+  return cleaned;
+}
+
+// Function to generate LLM text content in dory.json order, then append any extra files
+function generateLlmContent(docsDir: string): string {
+  const navOrder = getNavigationOrder(docsDir);
+  const mdxFilesMap = findMdxFilesMap(docsDir, docsDir);
+  const usedKeys = new Set<string>();
+  let aggregatedContent = '';
+
+  // Output files in navigation order, in order as in dory.json
+  for (const navKey of navOrder) {
+    // Try both with and without "index"
+    let fileKey = navKey;
+    if (mdxFilesMap[fileKey]) {
+      usedKeys.add(fileKey);
+    } else if (mdxFilesMap[`${fileKey}/index`]) {
+      fileKey = `${fileKey}/index`;
+      usedKeys.add(fileKey);
+    } else {
+      continue;
+    }
+    const filePath = mdxFilesMap[fileKey];
+    const rawContent = fs.readFileSync(filePath, 'utf-8');
+    const { data, content } = parseFrontmatter(rawContent);
+    const cleanedContent = cleanMdxContent(content);
+
+    // Add file header
+    aggregatedContent += `\n# ${fileKey}.mdx\n`;
+
+    // Add title if available
+    if (data.title) {
+      aggregatedContent += `## ${data.title}\n\n`;
+    }
+
+    // Add description if available
+    if (data.description) {
+      aggregatedContent += `${data.description}\n\n`;
+    }
+
+    // Add cleaned content
+    aggregatedContent += cleanedContent + '\n\n';
+    aggregatedContent += '---\n\n';
+  }
+
+  // Output any extra files not in navigation order, sorted
+  const extraKeys = Object.keys(mdxFilesMap).filter(k => !usedKeys.has(k)).sort();
+  for (const fileKey of extraKeys) {
+    const filePath = mdxFilesMap[fileKey];
+    const rawContent = fs.readFileSync(filePath, 'utf-8');
+    const { data, content } = parseFrontmatter(rawContent);
+    const cleanedContent = cleanMdxContent(content);
+
+    aggregatedContent += `\n# ${fileKey}.mdx\n`;
+    if (data.title) {
+      aggregatedContent += `## ${data.title}\n\n`;
+    }
+    if (data.description) {
+      aggregatedContent += `${data.description}\n\n`;
+    }
+    aggregatedContent += cleanedContent + '\n\n';
+    aggregatedContent += '---\n\n';
+  }
+
+  // Remove leading newlines for the very first file, so the first file is at the very top
+  return aggregatedContent.replace(/^\n+/, '').trim();
+}
+
+export function llmTxtGenerator(options: LlmTxtGeneratorOptions = {}): Plugin {
+  return {
+    name: 'llm-txt-generator',
+    generateBundle() {
+      const docsDir = options.docsDir || path.join(process.cwd(), 'docs');
+      const aggregatedContent = generateLlmContent(docsDir);
+
+      this.emitFile({
+        type: 'asset',
+        fileName: 'llms.txt',
+        source: aggregatedContent
+      });
+
+      // For logging, count the number of MDX files processed
+      const mdxFilesMap = findMdxFilesMap(docsDir, docsDir);
+      const mdxFilesCount = Object.keys(mdxFilesMap).length;
+
+      console.log(`✅ Generated LLM text file`);
+      console.log(`📄 Processed ${mdxFilesCount} MDX files`);
+      console.log(`📝 Total content size: ${aggregatedContent.length} characters`);
+    }
+  };
+}

package/src/plugins/sanitize.ts

@@ -0,0 +1,163 @@
+import { visit } from 'unist-util-visit';
+
+// This plugin replaces mdxTextExpression nodes with inlineCode for safety.
+export const remarkSafeVars = () => {
+  return (tree: any) => {
+    visit(tree, 'mdxTextExpression', (node: any, index: number | undefined, parent: any) => {
+      if (parent && typeof index === 'number') {
+        parent.children[index] = {
+          type: 'inlineCode',
+          value: node.value
+        };
+      }
+    });
+  };
+};
+
+const KNOWN_COMPONENTS = [
+  // Text components
+  'h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'p', 'Row', 'Col', 'a', 'img',
+  // Code components  
+  'code', 'CodeGroup', 'pre',
+  // UI components
+  'Accordion', 'AccordionGroup', 'Icon', 'Card', 'CardGroup',
+  'Callout', 'Note', 'Warning', 'Info', 'Tip', 'Check',
+  'Latex', 'API', 'Expandable', 'ResponseField', 'Properties', 'Property',
+  'Steps', 'Step', 'table', 'th', 'td', 'ul', 'ol', 'li',
+  'APIPlayground', 'WebSocketPlayground', 'AsyncAPI', 'Source',
+  // Common HTML elements that should be preserved
+  'div', 'span', 'br', 'hr', 'strong', 'em', 'b', 'i', 'u', 's',
+  'blockquote', 'nav', 'section', 'article', 'header', 'footer', 'main',
+  'aside', 'figure', 'figcaption', 'details', 'summary', 'mark', 'small',
+  'del', 'ins', 'sub', 'sup', 'kbd', 'samp', 'var', 'time', 'abbr',
+  'address', 'cite', 'q', 'dfn'
+];
+
+// Helper: returns true if the given code string is inside a code block (```)
+function isInsideCodeBlock(code: string, idx: number): boolean {
+  // Count the number of ``` before idx
+  const before = code.slice(0, idx);
+  const codeBlockMatches = before.match(/```/g);
+  // Odd number of ``` means inside a code block
+  return codeBlockMatches ? codeBlockMatches.length % 2 === 1 : false;
+}
+
+// Helper: checks if a known component tag has a matching closing tag
+function hasMatchingClosingTag(code: string, tagName: string, openingTagOffset: number): boolean {
+  // Simple approach: look for the next occurrence of either an opening or closing tag of the same type
+  const afterOpening = code.slice(openingTagOffset + `<${tagName}>`.length);
+  
+  // Find the first occurrence of either opening or closing tag
+  const nextOpeningMatch = afterOpening.match(new RegExp(`<${tagName}>`, 'i'));
+  const nextClosingMatch = afterOpening.match(new RegExp(`</${tagName}>`, 'i'));
+  
+  // If no closing tag found at all
+  if (!nextClosingMatch) {
+    return false;
+  }
+  
+  const nextOpeningIndex = nextOpeningMatch ? nextOpeningMatch.index! : Infinity;
+  const nextClosingIndex = nextClosingMatch ? nextClosingMatch.index! : Infinity;
+  
+  // If the closing tag comes before any other opening tag, this tag is properly closed
+  if (nextClosingIndex < nextOpeningIndex) {
+    // Check if the closing tag is inside a code block
+    const closingTagOffset = openingTagOffset + `<${tagName}>`.length + nextClosingIndex;
+    if (isInsideCodeBlock(code, closingTagOffset)) {
+      return false;
+    }
+    return true;
+  }
+  
+  return false; // Another opening tag comes before the closing tag, so this one is not properly closed
+}
+
+// Allow any <[a-z][a-z0-9]*> tag inside code blocks to be left as-is (not parsed as JSX).
+export function preprocessMdxTags() {
+  return {
+    name: 'preprocess-mdx-tags',
+    enforce: 'pre' as const,
+    transform(code: string, id: string) {
+      if (!id.endsWith('.mdx')) return;
+
+      // Remove leading whitespace from code fence blocks
+      let processed = code.replace(/^(\s+)(```\w*)/gm, '$2');
+
+      // Handle self-closing tags first (like <VideoPlayer />)
+      processed = processed.replace(/<([A-Za-z][A-Za-z0-9]*)[^>]*\/>/g, (match, tag, offset) => {
+        // If inside a code block, do not touch
+        if (isInsideCodeBlock(processed, offset)) {
+          return match;
+        }
+        
+        // If unknown component, treat as text
+        if (!KNOWN_COMPONENTS.includes(tag)) {
+          return `\`${match}\``;
+        }
+        return match;
+      });
+
+      // Only replace unrecognized <someTag> with `"<someTag>"` if NOT inside a code block
+      // This regex finds all <tag> occurrences (fixed to handle uppercase)
+      processed = processed.replace(/<([A-Za-z][A-Za-z0-9]*)>/g, (match, tag, offset) => {
+        // If inside a code block, do not touch
+        if (isInsideCodeBlock(processed, offset)) {
+          return match;
+        }
+        
+        // If known component, check if it has a closing tag
+        if (KNOWN_COMPONENTS.includes(tag)) {
+          // If it doesn't have a matching closing tag, treat it as text
+          if (!hasMatchingClosingTag(processed, tag, offset)) {
+            return `\`<${tag}>\``;
+          }
+          // Otherwise, leave as-is for proper JSX parsing
+          return match;
+        }
+        // For unknown components, always treat as text (wrap in backticks)
+        // If already inside backticks, do not wrap again
+        const before = processed.slice(0, offset);
+        const backtickMatches = before.match(/`+/g);
+        const insideBackticks = backtickMatches ? backtickMatches.reduce((acc, s) => acc + s.length, 0) % 2 === 1 : false;
+        if (insideBackticks) {
+          return match;
+        }
+        return `\`<${tag}>\``;
+      });
+
+      // This regex finds all <tag> and </tag> occurrences (fixed to handle uppercase)
+      processed = processed.replace(/<\/?([A-Za-z][A-Za-z0-9]*)>/g, (match, tag, offset) => {
+        // If inside a code block, do not touch
+        if (isInsideCodeBlock(processed, offset)) {
+          return match;
+        }
+        
+        // If known component, check if it has a closing tag (only for opening tags)
+        if (KNOWN_COMPONENTS.includes(tag)) {
+          // Only check for matching closing tag for opening tags
+          if (!match.startsWith('</')) {
+            if (!hasMatchingClosingTag(processed, tag, offset)) {
+              return `\`${match}\``;
+            }
+          }
+          // Otherwise, leave as-is for proper JSX parsing
+          return match;
+        }
+        // For unknown components, always treat as text (wrap in backticks)
+        // If already inside backticks, do not wrap again
+        const before = processed.slice(0, offset);
+        const backtickMatches = before.match(/`+/g);
+        const insideBackticks = backtickMatches ? backtickMatches.reduce((acc, s) => acc + s.length, 0) % 2 === 1 : false;
+        if (insideBackticks) {
+          return match;
+        }
+        return `\`${match}\``;
+      });
+
+      return {
+        code: processed,
+        map: null,
+      };
+    },
+  };
+}

package/src/plugins/unified-mdx-plugin.ts

@@ -0,0 +1,35 @@
+import type { Plugin } from 'vite';
+import { preprocessMdxContent } from '../mdx/processor';
+
+/**
+ * Unified MDX preprocessing plugin for Vite.
+ * This plugin ensures that ALL MDX preprocessing goes through the same pipeline
+ * as the CLI verify:content command.
+ * 
+ * This is the SINGLE SOURCE OF TRUTH for MDX preprocessing.
+ * Any changes to MDX processing should be made in src/mdx/processor.ts
+ */
+export function unifiedMdxPreprocessor(): Plugin {
+  return {
+    name: 'unified-mdx-preprocessor',
+    enforce: 'pre', // Must run before the MDX plugin
+    async transform(code: string, id: string) {
+      if (!id.endsWith('.mdx')) return;
+      
+      // Process through the unified preprocessing pipeline
+      const result = await preprocessMdxContent(code, id);
+
+      if (!result.success) {
+        // Throw the error to stop the build
+        throw result.error;
+      }
+
+      // Return the preprocessed content
+      // The actual MDX compilation will be handled by @mdx-js/rollup plugin
+      return {
+        code: result.processedContent || code,
+        map: null
+      };
+    }
+  };
+}

package/src/sdk/index.ts

@@ -0,0 +1,89 @@
+/**
+ * Dory Embedded Documentation SDK
+ * Export all framework integrations
+ */
+
+// Re-export React SDK
+export { useDoryDocs as useReactDoryDocs, DoryDocsTrigger, DoryDocsProvider } from './react';
+
+// Re-export Vue SDK
+export { useDoryDocs as useVueDoryDocs } from './vue';
+
+// Export types
+export type { DoryEmbedConfig } from '../embed/types';
+
+// Export vanilla JS API reference
+export interface DoryDocsAPI {
+  init(config: Partial<any>): DoryDocsAPI;
+  open(path?: string): Promise<void>;
+  close(): Promise<void>;
+  toggle(): Promise<void>;
+  navigate(path: string): Promise<void>;
+  setTheme(theme: 'light' | 'dark'): Promise<void>;
+  isOpen(): boolean;
+  destroy(): Promise<void>;
+  on(event: string, handler: Function): DoryDocsAPI;
+}
+
+/**
+ * Check if DoryDocs is loaded
+ */
+export function isDoryDocsLoaded(): boolean {
+  return typeof window !== 'undefined' && !!window.DoryDocs;
+}
+
+/**
+ * Wait for DoryDocs to be loaded
+ */
+export function waitForDoryDocs(timeout = 5000): Promise<boolean> {
+  return new Promise((resolve) => {
+    if (isDoryDocsLoaded()) {
+      resolve(true);
+      return;
+    }
+
+    const timeoutId = setTimeout(() => {
+      window.removeEventListener('dory:ready', handleReady);
+      resolve(false);
+    }, timeout);
+
+    const handleReady = () => {
+      clearTimeout(timeoutId);
+      resolve(true);
+    };
+
+    window.addEventListener('dory:ready', handleReady, { once: true });
+  });
+}
+
+/**
+ * Load Dory embed script dynamically
+ */
+export function loadDoryScript(docsUrl: string): Promise<void> {
+  return new Promise((resolve, reject) => {
+    if (isDoryDocsLoaded()) {
+      resolve();
+      return;
+    }
+
+    const script = document.createElement('script');
+    script.src = `${docsUrl}/embed.js`;
+    script.async = true;
+
+    script.onload = () => {
+      waitForDoryDocs().then((loaded) => {
+        if (loaded) {
+          resolve();
+        } else {
+          reject(new Error('DoryDocs failed to initialize'));
+        }
+      });
+    };
+
+    script.onerror = () => {
+      reject(new Error('Failed to load DoryDocs script'));
+    };
+
+    document.head.appendChild(script);
+  });
+}