doc-fetch-cli 2.0.4 → 2.0.6

package/website/src/routes/blog/[slug]/+page.ts

@@ -0,0 +1,53 @@
+// SSR Load function for proper SEO
+import type { PageLoad } from './$types';
+
+export const load: PageLoad = async ({ params }) => {
+	const posts = {
+		'convert-docs-to-markdown-for-llm': {
+			slug: 'convert-docs-to-markdown-for-llm',
+			title: 'How to Convert Documentation to Markdown for LLMs (Complete Guide)',
+			excerpt: 'Step-by-step guide: Transform entire documentation websites into clean, AI-ready markdown. Includes tools, techniques, and best practices for optimal LLM context.',
+			date: '2026-02-21',
+			author: 'AlphaTechini',
+			readTime: '8 min read',
+			tags: ['Tutorial', 'LLM', 'Markdown'],
+			category: 'rag',
+			subcategory: 'context-preparation',
+			modifiedDate: '2026-02-21',
+			relatedPosts: [
+				'llm-txt-index-guide',
+				'ai-agent-documentation-problem',
+				'best-practices-rag-context-preparation'
+			],
+			faqs: [
+				{
+					question: 'What is the best way to convert documentation to markdown?',
+					answer: 'Automated tools like DocFetch are most efficient. They crawl entire documentation sites, extract clean content, and generate structured markdown with semantic indexing (llm.txt).'
+				},
+				{
+					question: 'Why convert documentation to markdown for LLMs?',
+					answer: 'LLMs need complete context in a single prompt. Markdown provides clean, structured text without HTML bloat, navigation, or ads that waste tokens.'
+				},
+				{
+					question: 'What is llm.txt and how does it help?',
+					answer: 'llm.txt is a semantic index file that categorizes documentation sections (GUIDE, API, TUTORIAL) with descriptions. It helps AI agents navigate large documentation efficiently.'
+				},
+				{
+					question: 'Can I automate documentation conversion?',
+					answer: 'Yes. Tools like DocFetch can automatically fetch, clean, and convert entire documentation sites with one command. Set up cron jobs for regular updates.'
+				}
+			],
+			content: '' // Will be loaded dynamically
+		}
+	};
+	
+	const post = posts[params.slug as keyof typeof posts];
+	
+	if (!post) {
+		throw new Error('Post not found');
+	}
+	
+	return {
+		post
+	};
+};

package/website/src/routes/sitemap.xml/+server.ts

@@ -0,0 +1,62 @@
+// Dynamic sitemap generation for SEO
+import type { RequestHandler } from '@sveltejs/kit';
+
+export const GET: RequestHandler = async () => {
+	// In production, fetch from CMS/markdown files
+	const baseUrl = 'https://docfetch.dev';
+	
+	const staticPages = [
+		{ path: '/', priority: '1.0', changefreq: 'weekly' },
+		{ path: '/#features', priority: '0.8', changefreq: 'monthly' },
+		{ path: '/#installation', priority: '0.8', changefreq: 'monthly' },
+		{ path: '/#usage', priority: '0.7', changefreq: 'monthly' },
+		{ path: '/blog', priority: '0.9', changefreq: 'daily' }
+	];
+	
+	const blogPosts = [
+		{
+			slug: 'convert-docs-to-markdown-for-llm',
+			path: '/rag/context-preparation/convert-docs-to-markdown',
+			lastmod: '2026-02-21',
+			priority: '0.8',
+			changefreq: 'weekly'
+		},
+		{
+			slug: 'llm-txt-index-guide',
+			path: '/rag/context-preparation/llm-txt-guide',
+			lastmod: '2026-02-21',
+			priority: '0.7',
+			changefreq: 'weekly'
+		},
+		{
+			slug: 'ai-agent-documentation-problem',
+			path: '/llm-tools/documentation/ai-agents-cant-read-docs',
+			lastmod: '2026-02-21',
+			priority: '0.7',
+			changefreq: 'weekly'
+		}
+	];
+	
+	const allPages = [...staticPages, ...blogPosts];
+	
+	const sitemap = `<?xml version="1.0" encoding="UTF-8"?>
+<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"
+        xmlns:news="http://www.google.com/schemas/sitemap-news/0.9"
+        xmlns:xhtml="http://www.w3.org/1999/xhtml"
+        xmlns:image="http://www.google.com/schemas/sitemap-image/1.1"
+        xmlns:video="http://www.google.com/schemas/sitemap-video/1.1">
+${allPages.map(page => `  <url>
+    <loc>${baseUrl}${page.path}</loc>
+    <lastmod>${page.lastmod || new Date().toISOString().split('T')[0]}</lastmod>
+    <changefreq>${page.changefreq}</changefreq>
+    <priority>${page.priority}</priority>
+  </url>`).join('\n')}
+</urlset>`;
+	
+	return new Response(sitemap, {
+		headers: {
+			'Content-Type': 'application/xml',
+			'Cache-Control': 'max-age=0, must-revalidate'
+		}
+	});
+};

package/website/static/favicon.svg

@@ -0,0 +1,10 @@
+<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 100 100">
+  <defs>
+    <linearGradient id="grad" x1="0%" y1="0%" x2="100%" y2="100%">
+      <stop offset="0%" style="stop-color:#0066cc;stop-opacity:1" />
+      <stop offset="100%" style="stop-color:#0052a3;stop-opacity:1" />
+    </linearGradient>
+  </defs>
+  <rect width="100" height="100" rx="20" fill="url(#grad)"/>
+  <text x="50" y="70" font-family="Georgia, serif" font-size="60" font-weight="bold" fill="white" text-anchor="middle">D</text>
+</svg>

package/website/static/og.png

@@ -0,0 +1,2 @@
+<!-- Placeholder for OG image - replace with actual 1200x630px PNG -->
+<!-- Recommended: Create with Figma/Canva showing DocFetch logo + tagline -->

package/website/static/og.svg

@@ -0,0 +1,26 @@
+<svg width="1200" height="630" xmlns="http://www.w3.org/2000/svg">
+  <defs>
+    <linearGradient id="bg" x1="0%" y1="0%" x2="100%" y2="100%">
+      <stop offset="0%" stop-color="#0066cc"/>
+      <stop offset="100%" stop-color="#004080"/>
+    </linearGradient>
+  </defs>
+  
+  <!-- Background -->
+  <rect width="1200" height="630" fill="url(#bg)"/>
+  
+  <!-- Icon -->
+  <text x="150" y="400" font-family="Georgia, serif" font-size="280" fill="white">📚</text>
+  
+  <!-- Title -->
+  <text x="450" y="280" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, sans-serif" font-size="72" font-weight="700" fill="white">DocFetch</text>
+  
+  <!-- Tagline -->
+  <text x="450" y="360" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, sans-serif" font-size="36" fill="#e0e0e0">Transform documentation into AI-ready markdown</text>
+  
+  <!-- URL -->
+  <text x="450" y="450" font-family="monospace" font-size="28" fill="#b3d9ff">docfetch.dev</text>
+  
+  <!-- Features -->
+  <text x="450" y="520" font-family="-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, sans-serif" font-size="24" fill="#cce5ff">Single-file markdown • LLM.txt indexing • One command</text>
+</svg>

package/website/static/robots.txt

@@ -0,0 +1,43 @@
+# Robots.txt for DocFetch
+# Allow all crawlers full access
+
+User-agent: *
+Allow: /
+
+# Sitemap location
+Sitemap: https://docfetch.dev/sitemap.xml
+
+# Crawl-delay (optional, be nice to servers)
+Crawl-delay: 1
+
+# Block common bad paths
+Disallow: /_app/
+Disallow: /node_modules/
+Disallow: /*.json$
+
+# Allow all blog content
+Allow: /blog/
+Allow: /rag/
+Allow: /llm-tools/
+Allow: /web3/
+Allow: /ai-infra/
+
+# Specific rules for Googlebot (priority crawler)
+User-agent: Googlebot
+Allow: /
+Crawl-delay: 0
+
+# Bing
+User-agent: Bingbot
+Allow: /
+Crawl-delay: 2
+
+# Block AI scrapers if desired (controversial but option)
+# User-agent: GPTBot
+# Disallow: /
+
+# User-agent: ChatGPT-User
+# Disallow: /
+
+# User-agent: CCBot
+# Disallow: /

package/website/svelte.config.js

@@ -0,0 +1,13 @@
+import adapter from '@sveltejs/adapter-auto';
+
+/** @type {import('@sveltejs/kit').Config} */
+const config = {
+	kit: {
+		// adapter-auto only supports some environments, see https://svelte.dev/docs/kit/adapter-auto for a list.
+		// If your environment is not supported, or you settled on a specific environment, switch out the adapter.
+		// See https://svelte.dev/docs/kit/adapters for more information about adapters.
+		adapter: adapter()
+	}
+};
+
+export default config;

package/website/tsconfig.json

@@ -0,0 +1,20 @@
+{
+	"extends": "./.svelte-kit/tsconfig.json",
+	"compilerOptions": {
+		"rewriteRelativeImportExtensions": true,
+		"allowJs": true,
+		"checkJs": true,
+		"esModuleInterop": true,
+		"forceConsistentCasingInFileNames": true,
+		"resolveJsonModule": true,
+		"skipLibCheck": true,
+		"sourceMap": true,
+		"strict": true,
+		"moduleResolution": "bundler"
+	}
+	// Path aliases are handled by https://svelte.dev/docs/kit/configuration#alias
+	// except $lib which is handled by https://svelte.dev/docs/kit/configuration#files
+	//
+	// To make changes to top-level options such as include and exclude, we recommend extending
+	// the generated config; see https://svelte.dev/docs/kit/configuration#typescript
+}

package/website/vite.config.ts

@@ -0,0 +1,6 @@
+import { sveltekit } from '@sveltejs/kit/vite';
+import { defineConfig } from 'vite';
+
+export default defineConfig({
+	plugins: [sveltekit()]
+});