@clidey/dory 0.4.0 → 0.6.0

package/README.md

@@ -150,6 +150,16 @@ This will create a `dist` directory containing the fully built static site, read
 
 ## 🐳 Docker Deployment (Optional)
 
+You can use our official Docker image `clidey/dory` to create a static server. Simply create a Dockerfile inside the project and build the following Dockerfile:
+
+```dockerfile
+FROM clidey/dory
+WORKDIR /app
+COPY . .
+RUN dory build
+CMD ["dory", "preview"]
+```
+
 You can also build and run the static site inside a Docker container using Nginx:
 
 ### Build Docker Image

package/k8s/Dockerfile

@@ -1,15 +1,11 @@
-FROM node:20-alpine AS builder
+FROM node:22.16-alpine
 
-RUN corepack enable && corepack prepare pnpm@10.10.0 --activate
 WORKDIR /app
-COPY package.json pnpm-lock.yaml ./
-RUN pnpm install --frozen-lockfile
-COPY . .
-RUN pnpm build
 
-FROM nginx:alpine
-COPY --from=builder /app/dist /usr/share/nginx/html
-COPY --from=builder /app/k8s/config/nginx-default.conf /etc/nginx/conf.d/default.conf
-EXPOSE 80
+COPY ./k8s/config/nginx-default.conf /app/k8s/config/nginx-default.conf
 
-CMD ["nginx", "-g", "daemon off;"] 
+# Install dory globally
+RUN npm install -g @clidey/dory
+EXPOSE 3000
+
+CMD ["dory", "preview"]

package/k8s/docs.Dockerfile

@@ -0,0 +1,15 @@
+FROM node:20-alpine AS builder
+
+RUN corepack enable && corepack prepare pnpm@10.10.0 --activate
+WORKDIR /app
+COPY package.json pnpm-lock.yaml ./
+RUN pnpm install --frozen-lockfile
+COPY . .
+RUN pnpm build
+
+FROM nginx:alpine
+COPY --from=builder /app/dist /usr/share/nginx/html
+COPY --from=builder /app/k8s/config/nginx-default.conf /etc/nginx/conf.d/default.conf
+EXPOSE 80
+
+CMD ["nginx", "-g", "daemon off;"] 

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@clidey/dory",
   "private": false,
-  "version": "0.4.0",
+  "version": "0.6.0",
   "type": "module",
   "description": "A CLI tool for building and previewing documentation sites",
   "keywords": [
@@ -66,5 +66,8 @@
     "vite-plugin-mdx": "^3.6.1",
     "wouter-preact": "^3.7.1"
   },
-  "packageManager": "pnpm@10.10.0+sha512.d615db246fe70f25dcfea6d8d73dee782ce23e2245e3c4f6f888249fb568149318637dca73c2c5c8ef2a4ca0d5657fb9567188bfab47f566d1ee6ce987815c39"
+  "packageManager": "pnpm@10.10.0+sha512.d615db246fe70f25dcfea6d8d73dee782ce23e2245e3c4f6f888249fb568149318637dca73c2c5c8ef2a4ca0d5657fb9567188bfab47f566d1ee6ce987815c39",
+  "devDependencies": {
+    "@types/node": "^24.0.12"
+  }
 }

package/scripts/generate-llm-txt.js

@@ -0,0 +1,115 @@
+#!/usr/bin/env node
+
+import fs from 'fs';
+import path from 'path';
+import { fileURLToPath } from 'url';
+
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = path.dirname(__filename);
+
+// Function to parse frontmatter from MDX content
+function parseFrontmatter(content) {
+  const frontmatterMatch = content.match(/^---\n([\s\S]*?)\n---\n/);
+  if (!frontmatterMatch) return { data: {}, content };
+  
+  const frontmatter = frontmatterMatch[1];
+  const data = {};
+  
+  frontmatter.split('\n').forEach(line => {
+    const [key, ...valueParts] = line.split(':');
+    if (key && valueParts.length) {
+      const value = valueParts.join(':').trim();
+      data[key.trim()] = value.replace(/^["']|["']$/g, '');
+    }
+  });
+
+  return { data, content: content.slice(frontmatterMatch[0].length) };
+}
+
+// Function to recursively find all MDX files
+function findMdxFiles(dir) {
+  const files = [];
+  const items = fs.readdirSync(dir, { withFileTypes: true });
+  
+  for (const item of items) {
+    const fullPath = path.join(dir, item.name);
+    if (item.isDirectory()) {
+      files.push(...findMdxFiles(fullPath));
+    } else if (item.isFile() && item.name.endsWith('.mdx')) {
+      files.push(fullPath);
+    }
+  }
+  
+  return files;
+}
+
+// Function to clean MDX content for LLM consumption
+function cleanMdxContent(content) {
+  // Remove JSX components but keep their content
+  let cleaned = content
+    // Remove JSX opening/closing tags but keep content
+    .replace(/<(\w+)([^>]*?)>/g, '')
+    .replace(/<\/\w+>/g, '')
+    // Remove code block language specifiers
+    .replace(/```(\w+)/g, '```')
+    // Remove HTML comments
+    .replace(/<!--[\s\S]*?-->/g, '')
+    // Remove excessive whitespace
+    .replace(/\n\s*\n\s*\n/g, '\n\n')
+    // Remove leading/trailing whitespace
+    .trim();
+    
+  return cleaned;
+}
+
+// Main function to generate LLM text
+async function generateLlmTxt() {
+  const docsDir = path.join(__dirname, '..', 'docs');
+  const outputDir = path.join(__dirname, '..', 'dist');
+  
+  // Ensure output directory exists
+  if (!fs.existsSync(outputDir)) {
+    fs.mkdirSync(outputDir, { recursive: true });
+  }
+  
+  const mdxFiles = findMdxFiles(docsDir);
+  let aggregatedContent = '';
+  
+  // Sort files for consistent output
+  mdxFiles.sort();
+  
+  for (const filePath of mdxFiles) {
+    const rawContent = fs.readFileSync(filePath, 'utf-8');
+    const { data, content } = parseFrontmatter(rawContent);
+    const cleanedContent = cleanMdxContent(content);
+    
+    // Add file header
+    const relativePath = path.relative(docsDir, filePath);
+    aggregatedContent += `\n# ${relativePath}\n`;
+    
+    // Add title if available
+    if (data.title) {
+      aggregatedContent += `## ${data.title}\n\n`;
+    }
+    
+    // Add description if available
+    if (data.description) {
+      aggregatedContent += `${data.description}\n\n`;
+    }
+    
+    // Add cleaned content
+    aggregatedContent += cleanedContent + '\n\n';
+    aggregatedContent += '---\n\n';
+  }
+  
+  // Write to output file
+  const outputPath = path.join(outputDir, 'llms.txt');
+  fs.writeFileSync(outputPath, aggregatedContent.trim());
+  
+  console.log(`✅ Generated LLM text file: ${outputPath}`);
+  console.log(`📄 Processed ${mdxFiles.length} MDX files`);
+  console.log(`📝 Total content size: ${aggregatedContent.length} characters`);
+}
+
+// Run the script
+generateLlmTxt().catch(console.error);

package/src/plugins/llm-txt-dev-server.ts

@@ -0,0 +1,183 @@
+import type { Plugin } from 'vite';
+import fs from 'fs';
+import path from 'path';
+
+// Read dory.json to get navigation order (flat list, in order)
+function getNavigationOrder(docsDir: string): string[] {
+  const doryJsonPath = path.join(docsDir, 'dory.json');
+  if (!fs.existsSync(doryJsonPath)) return [];
+  const doryJson = JSON.parse(fs.readFileSync(doryJsonPath, 'utf-8'));
+  const order: string[] = [];
+
+  function walkPages(pages: any[]) {
+    for (const page of pages) {
+      if (typeof page === 'string') {
+        order.push(page);
+      } else if (typeof page === 'object' && page.pages) {
+        walkPages(page.pages);
+      }
+    }
+  }
+
+  if (doryJson.navigation && Array.isArray(doryJson.navigation.tabs)) {
+    for (const tab of doryJson.navigation.tabs) {
+      if (tab.groups && Array.isArray(tab.groups)) {
+        for (const group of tab.groups) {
+          if (group.pages && Array.isArray(group.pages)) {
+            walkPages(group.pages);
+          }
+        }
+      }
+    }
+  }
+  return order;
+}
+
+// Function to parse frontmatter from MDX content
+function parseFrontmatter(content: string) {
+  const frontmatterMatch = content.match(/^---\n([\s\S]*?)\n---\n/);
+  if (!frontmatterMatch) return { data: {}, content };
+
+  const frontmatter = frontmatterMatch[1];
+  const data: Record<string, string> = {};
+
+  frontmatter.split('\n').forEach(line => {
+    const [key, ...valueParts] = line.split(':');
+    if (key && valueParts.length) {
+      const value = valueParts.join(':').trim();
+      data[key.trim()] = value.replace(/^["']|["']$/g, '');
+    }
+  });
+
+  return { data, content: content.slice(frontmatterMatch[0].length) };
+}
+
+// Function to recursively find all MDX files and map by relative path (without .mdx)
+function findMdxFilesMap(dir: string, baseDir: string): Record<string, string> {
+  const files: Record<string, string> = {};
+  const items = fs.readdirSync(dir, { withFileTypes: true });
+
+  for (const item of items) {
+    const fullPath = path.join(dir, item.name);
+    if (item.isDirectory()) {
+      Object.assign(files, findMdxFilesMap(fullPath, baseDir));
+    } else if (item.isFile() && item.name.endsWith('.mdx')) {
+      // Key: relative path without .mdx, with forward slashes
+      let rel = path.relative(baseDir, fullPath).replace(/\\/g, '/');
+      if (rel.endsWith('.mdx')) rel = rel.slice(0, -4);
+      files[rel] = fullPath;
+    }
+  }
+
+  return files;
+}
+
+// Function to clean MDX content for LLM consumption
+function cleanMdxContent(content: string): string {
+  // Remove JSX components but keep their content
+  let cleaned = content
+    // Remove JSX opening/closing tags but keep content
+    .replace(/<(\w+)([^>]*?)>/g, '')
+    .replace(/<\/\w+>/g, '')
+    // Remove code block language specifiers
+    .replace(/```(\w+)/g, '```')
+    // Remove HTML comments
+    .replace(/<!--[\s\S]*?-->/g, '')
+    // Remove excessive whitespace
+    .replace(/\n\s*\n\s*\n/g, '\n\n')
+    // Remove leading/trailing whitespace
+    .trim();
+
+  return cleaned;
+}
+
+// Function to generate LLM text content in dory.json order, then append any extra files
+function generateLlmContent(docsDir: string): string {
+  const navOrder = getNavigationOrder(docsDir);
+  const mdxFilesMap = findMdxFilesMap(docsDir, docsDir);
+  const usedKeys = new Set<string>();
+  let aggregatedContent = '';
+
+  // Output files in navigation order, in order as in dory.json
+  for (const navKey of navOrder) {
+    // Try both with and without "index"
+    let fileKey = navKey;
+    if (mdxFilesMap[fileKey]) {
+      usedKeys.add(fileKey);
+    } else if (mdxFilesMap[`${fileKey}/index`]) {
+      fileKey = `${fileKey}/index`;
+      usedKeys.add(fileKey);
+    } else {
+      continue;
+    }
+    const filePath = mdxFilesMap[fileKey];
+    const rawContent = fs.readFileSync(filePath, 'utf-8');
+    const { data, content } = parseFrontmatter(rawContent);
+    const cleanedContent = cleanMdxContent(content);
+
+    // Add file header
+    aggregatedContent += `\n# ${fileKey}.mdx\n`;
+
+    // Add title if available
+    if (data.title) {
+      aggregatedContent += `## ${data.title}\n\n`;
+    }
+
+    // Add description if available
+    if (data.description) {
+      aggregatedContent += `${data.description}\n\n`;
+    }
+
+    // Add cleaned content
+    aggregatedContent += cleanedContent + '\n\n';
+    aggregatedContent += '---\n\n';
+  }
+
+  // Output any extra files not in navigation order, sorted
+  const extraKeys = Object.keys(mdxFilesMap).filter(k => !usedKeys.has(k)).sort();
+  for (const fileKey of extraKeys) {
+    const filePath = mdxFilesMap[fileKey];
+    const rawContent = fs.readFileSync(filePath, 'utf-8');
+    const { data, content } = parseFrontmatter(rawContent);
+    const cleanedContent = cleanMdxContent(content);
+
+    aggregatedContent += `\n# ${fileKey}.mdx\n`;
+    if (data.title) {
+      aggregatedContent += `## ${data.title}\n\n`;
+    }
+    if (data.description) {
+      aggregatedContent += `${data.description}\n\n`;
+    }
+    aggregatedContent += cleanedContent + '\n\n';
+    aggregatedContent += '---\n\n';
+  }
+
+  // Remove leading newlines for the very first file, so the first file is at the very top
+  return aggregatedContent.replace(/^\n+/, '').trim();
+}
+
+export function llmTxtDevServer(): Plugin {
+  return {
+    name: 'llm-txt-dev-server',
+    configureServer(server) {
+      server.middlewares.use('/llms.txt', (req, res, next) => {
+        if (req.method !== 'GET') {
+          return next();
+        }
+
+        try {
+          const docsDir = path.join(process.cwd(), 'docs');
+          const llmContent = generateLlmContent(docsDir);
+
+          res.setHeader('Content-Type', 'text/plain; charset=utf-8');
+          res.setHeader('Content-Length', Buffer.byteLength(llmContent));
+          res.end(llmContent);
+        } catch (error) {
+          console.error('Error generating LLM text:', error);
+          res.statusCode = 500;
+          res.end('Internal Server Error');
+        }
+      });
+    }
+  };
+}

package/src/plugins/llm-txt-generator.ts

@@ -0,0 +1,186 @@
+import type { Plugin } from 'vite';
+import fs from 'fs';
+import path from 'path';
+
+interface LlmTxtGeneratorOptions {
+  docsDir?: string;
+  outputPath?: string;
+}
+
+// Read dory.json to get navigation order (flat list, in order)
+function getNavigationOrder(docsDir: string): string[] {
+  const doryJsonPath = path.join(docsDir, 'dory.json');
+  if (!fs.existsSync(doryJsonPath)) return [];
+  const doryJson = JSON.parse(fs.readFileSync(doryJsonPath, 'utf-8'));
+  const order: string[] = [];
+
+  function walkPages(pages: any[]) {
+    for (const page of pages) {
+      if (typeof page === 'string') {
+        order.push(page);
+      } else if (typeof page === 'object' && page.pages) {
+        walkPages(page.pages);
+      }
+    }
+  }
+
+  if (doryJson.navigation && Array.isArray(doryJson.navigation.tabs)) {
+    for (const tab of doryJson.navigation.tabs) {
+      if (tab.groups && Array.isArray(tab.groups)) {
+        for (const group of tab.groups) {
+          if (group.pages && Array.isArray(group.pages)) {
+            walkPages(group.pages);
+          }
+        }
+      }
+    }
+  }
+  return order;
+}
+
+// Function to parse frontmatter from MDX content
+function parseFrontmatter(content: string) {
+  const frontmatterMatch = content.match(/^---\n([\s\S]*?)\n---\n/);
+  if (!frontmatterMatch) return { data: {}, content };
+
+  const frontmatter = frontmatterMatch[1];
+  const data: Record<string, string> = {};
+
+  frontmatter.split('\n').forEach(line => {
+    const [key, ...valueParts] = line.split(':');
+    if (key && valueParts.length) {
+      const value = valueParts.join(':').trim();
+      data[key.trim()] = value.replace(/^["']|["']$/g, '');
+    }
+  });
+
+  return { data, content: content.slice(frontmatterMatch[0].length) };
+}
+
+// Function to recursively find all MDX files and map by relative path (without .mdx)
+function findMdxFilesMap(dir: string, baseDir: string): Record<string, string> {
+  const files: Record<string, string> = {};
+  const items = fs.readdirSync(dir, { withFileTypes: true });
+
+  for (const item of items) {
+    const fullPath = path.join(dir, item.name);
+    if (item.isDirectory()) {
+      Object.assign(files, findMdxFilesMap(fullPath, baseDir));
+    } else if (item.isFile() && item.name.endsWith('.mdx')) {
+      // Key: relative path without .mdx, with forward slashes
+      let rel = path.relative(baseDir, fullPath).replace(/\\/g, '/');
+      if (rel.endsWith('.mdx')) rel = rel.slice(0, -4);
+      files[rel] = fullPath;
+    }
+  }
+
+  return files;
+}
+
+// Function to clean MDX content for LLM consumption
+function cleanMdxContent(content: string): string {
+  // Remove JSX components but keep their content
+  let cleaned = content
+    // Remove JSX opening/closing tags but keep content
+    .replace(/<(\w+)([^>]*?)>/g, '')
+    .replace(/<\/\w+>/g, '')
+    // Remove code block language specifiers
+    .replace(/```(\w+)/g, '```')
+    // Remove HTML comments
+    .replace(/<!--[\s\S]*?-->/g, '')
+    // Remove excessive whitespace
+    .replace(/\n\s*\n\s*\n/g, '\n\n')
+    // Remove leading/trailing whitespace
+    .trim();
+
+  return cleaned;
+}
+
+// Function to generate LLM text content in dory.json order, then append any extra files
+function generateLlmContent(docsDir: string): string {
+  const navOrder = getNavigationOrder(docsDir);
+  const mdxFilesMap = findMdxFilesMap(docsDir, docsDir);
+  const usedKeys = new Set<string>();
+  let aggregatedContent = '';
+
+  // Output files in navigation order, in order as in dory.json
+  for (const navKey of navOrder) {
+    // Try both with and without "index"
+    let fileKey = navKey;
+    if (mdxFilesMap[fileKey]) {
+      usedKeys.add(fileKey);
+    } else if (mdxFilesMap[`${fileKey}/index`]) {
+      fileKey = `${fileKey}/index`;
+      usedKeys.add(fileKey);
+    } else {
+      continue;
+    }
+    const filePath = mdxFilesMap[fileKey];
+    const rawContent = fs.readFileSync(filePath, 'utf-8');
+    const { data, content } = parseFrontmatter(rawContent);
+    const cleanedContent = cleanMdxContent(content);
+
+    // Add file header
+    aggregatedContent += `\n# ${fileKey}.mdx\n`;
+
+    // Add title if available
+    if (data.title) {
+      aggregatedContent += `## ${data.title}\n\n`;
+    }
+
+    // Add description if available
+    if (data.description) {
+      aggregatedContent += `${data.description}\n\n`;
+    }
+
+    // Add cleaned content
+    aggregatedContent += cleanedContent + '\n\n';
+    aggregatedContent += '---\n\n';
+  }
+
+  // Output any extra files not in navigation order, sorted
+  const extraKeys = Object.keys(mdxFilesMap).filter(k => !usedKeys.has(k)).sort();
+  for (const fileKey of extraKeys) {
+    const filePath = mdxFilesMap[fileKey];
+    const rawContent = fs.readFileSync(filePath, 'utf-8');
+    const { data, content } = parseFrontmatter(rawContent);
+    const cleanedContent = cleanMdxContent(content);
+
+    aggregatedContent += `\n# ${fileKey}.mdx\n`;
+    if (data.title) {
+      aggregatedContent += `## ${data.title}\n\n`;
+    }
+    if (data.description) {
+      aggregatedContent += `${data.description}\n\n`;
+    }
+    aggregatedContent += cleanedContent + '\n\n';
+    aggregatedContent += '---\n\n';
+  }
+
+  // Remove leading newlines for the very first file, so the first file is at the very top
+  return aggregatedContent.replace(/^\n+/, '').trim();
+}
+
+export function llmTxtGenerator(options: LlmTxtGeneratorOptions = {}): Plugin {
+  return {
+    name: 'llm-txt-generator',
+    generateBundle() {
+      const docsDir = options.docsDir || path.join(process.cwd(), 'docs');
+      const aggregatedContent = generateLlmContent(docsDir);
+
+      this.emitFile({
+        type: 'asset',
+        fileName: 'llms.txt',
+        source: aggregatedContent
+      });
+
+      // For logging, count the number of MDX files processed
+      const mdxFilesMap = findMdxFilesMap(docsDir, docsDir);
+      const mdxFilesCount = Object.keys(mdxFilesMap).length;
+
+      console.log(`✅ Generated LLM text file`);
+      console.log(`📄 Processed ${mdxFilesCount} MDX files`);
+      console.log(`📝 Total content size: ${aggregatedContent.length} characters`);
+    }
+  };
+}

package/vite.config.ts

@@ -4,6 +4,8 @@ import mdx from '@mdx-js/rollup';
 import tailwindcss from '@tailwindcss/vite'
 import remarkGfm from 'remark-gfm';
 import { remarkSafeVars } from './src/plugins/sanitize';
+import { llmTxtGenerator } from './src/plugins/llm-txt-generator';
+import { llmTxtDevServer } from './src/plugins/llm-txt-dev-server';
 
 export default defineConfig({
   plugins: [
@@ -13,6 +15,8 @@ export default defineConfig({
       remarkPlugins: [remarkGfm, remarkSafeVars],
     }),
     tailwindcss(),
+    llmTxtGenerator(),
+    llmTxtDevServer(),
   ],
   resolve: {
     alias: {