doc-fetch-cli 1.0.2

package/pkg/fetcher/fetcher.go

@@ -0,0 +1,332 @@
+package fetcher
+
+import (
+	"fmt"
+	"log"
+	"net"
+	"net/http"
+	"net/url"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/PuerkitoBio/goquery"
+	"golang.org/x/net/html"
+)
+
+// Config holds the configuration for the documentation fetcher
+type Config struct {
+	BaseURL         string
+	OutputPath      string
+	MaxDepth        int
+	Workers         int
+	UserAgent       string
+	GenerateLLMTxt  bool
+}
+
+// Page represents a fetched documentation page
+type Page struct {
+	URL     string
+	Title   string
+	Content string
+	Links   []string
+}
+
+// LLMTxtEntry represents an entry in the llm.txt file
+type LLMTxtEntry struct {
+	Type        string
+	Title       string
+	URL         string
+	Description string
+}
+
+// Run executes the documentation fetching process
+func Run(config Config) error {
+	// Validate configuration
+	if err := validateConfig(&config); err != nil {
+		return fmt.Errorf("invalid configuration: %w", err)
+	}
+
+	log.Printf("Starting documentation fetch from: %s", config.BaseURL)
+	
+	// Create a visited map to avoid duplicate fetching
+	visited := make(map[string]bool)
+	var mutex sync.Mutex
+	
+	// Create channel for pages and results
+	pagesChan := make(chan *Page, config.Workers*2)
+	resultsChan := make(chan string, config.Workers*2)
+	var llmEntries []LLMTxtEntry
+	
+	// Start worker goroutines
+	var wg sync.WaitGroup
+	for i := 0; i < config.Workers; i++ {
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			worker(config, pagesChan, resultsChan, &mutex, visited, &llmEntries)
+		}()
+	}
+	
+	// Start the initial fetch
+	pagesChan <- &Page{URL: config.BaseURL, Title: "Root"}
+	
+	// Close pages channel when all workers are done
+	go func() {
+		wg.Wait()
+		close(pagesChan)
+		close(resultsChan)
+	}()
+	
+	// Collect results and write to file
+	err := writeResults(config.OutputPath, resultsChan)
+	if err != nil {
+		return err
+	}
+	
+	// Generate LLM.txt if requested
+	if config.GenerateLLMTxt {
+		llmTxtPath := strings.TrimSuffix(config.OutputPath, ".md") + ".llm.txt"
+		err = GenerateLLMTxt(llmEntries, llmTxtPath)
+		if err != nil {
+			log.Printf("Warning: Failed to generate llm.txt: %v", err)
+		} else {
+			log.Printf("LLM.txt generated: %s", llmTxtPath)
+		}
+	}
+	
+	return nil
+}
+
+// worker processes pages from the channel
+func worker(config Config, pagesChan <-chan *Page, resultsChan chan<- string, mutex *sync.Mutex, visited map[string]bool, llmEntries *[]LLMTxtEntry) {
+	client := &http.Client{
+		Timeout: 30 * time.Second,
+		// Add transport with security restrictions
+		Transport: &http.Transport{
+			DisableKeepAlives: true,
+		},
+	}
+	
+	for page := range pagesChan {
+		// Validate URL before fetching
+		if err := isValidURL(page.URL); err != nil {
+			log.Printf("Skipping invalid URL %s: %v", page.URL, err)
+			continue
+		}
+		
+		mutex.Lock()
+		if visited[page.URL] {
+			mutex.Unlock()
+			continue
+		}
+		visited[page.URL] = true
+		mutex.Unlock()
+		
+		log.Printf("Fetching: %s", page.URL)
+		
+		// Rate limiting - be respectful to servers
+		time.Sleep(100 * time.Millisecond)
+		
+		// Fetch the page
+		req, err := http.NewRequest("GET", page.URL, nil)
+		if err != nil {
+			log.Printf("Error creating request for %s: %v", page.URL, err)
+			continue
+		}
+		req.Header.Set("User-Agent", config.UserAgent)
+		
+		resp, err := client.Do(req)
+		if err != nil {
+			log.Printf("Error fetching %s: %v", page.URL, err)
+			continue
+		}
+		defer resp.Body.Close()
+		
+		if resp.StatusCode != 200 {
+			log.Printf("Non-200 status code %d for %s", resp.StatusCode, page.URL)
+			continue
+		}
+		
+		// Parse HTML
+		doc, err := goquery.NewDocumentFromReader(resp.Body)
+		if err != nil {
+			log.Printf("Error parsing HTML for %s: %v", page.URL, err)
+			continue
+		}
+		
+		// Extract title
+		title := doc.Find("title").Text()
+		if title == "" {
+			title = page.URL
+		}
+		
+		// Clean and extract content
+		content := cleanContent(doc)
+		if content == "" {
+			log.Printf("No content found for %s", page.URL)
+			continue
+		}
+		
+		// Send result to output
+		resultsChan <- fmt.Sprintf("## %s\n\n%s\n\n---\n\n", title, content)
+		
+		// Generate LLM.txt entry if requested
+		if config.GenerateLLMTxt {
+			cleanTitle := CleanTitle(title)
+			entryType := ClassifyPage(page.URL, cleanTitle)
+			description := ExtractDescription(content)
+			
+			entry := LLMTxtEntry{
+				Type:        entryType,
+				Title:       cleanTitle,
+				URL:         page.URL,
+				Description: description,
+			}
+			
+			mutex.Lock()
+			*llmEntries = append(*llmEntries, entry)
+			mutex.Unlock()
+		}
+		
+		// Extract links for further crawling (limited depth logic would go here)
+		// For MVP, we'll just fetch the main page
+		// Future: implement link extraction and recursive crawling
+	}
+}
+
+// cleanContent extracts and cleans the main documentation content
+func cleanContent(doc *goquery.Document) string {
+	// Common selectors for documentation content
+	selectors := []string{
+		"main",
+		"article",
+		".content",
+		".docs-content",
+		"#main-content",
+		".documentation",
+		".post-content",
+		".markdown-body",
+		".content-wrapper",
+		".doc-content",
+	}
+	
+	// Try each selector
+	for _, selector := range selectors {
+		if el := doc.Find(selector); el.Length() > 0 {
+			// Remove unwanted elements
+			el.Find("nav, header, footer, .sidebar, .toc, .navigation, script, style, .ad, .advertisement").Remove()
+			
+			// Convert to HTML and then clean
+			htmlContent, err := el.Html()
+			if err != nil {
+				continue
+			}
+			
+			// Basic HTML cleaning
+			cleaned := cleanHTML(htmlContent)
+			if cleaned != "" {
+				return cleaned
+			}
+		}
+	}
+	
+	// Fallback: try to get body content
+	body := doc.Find("body")
+	if body.Length() > 0 {
+		body.Find("nav, header, footer, .sidebar, .toc, .navigation, script, style, .ad, .advertisement").Remove()
+		htmlContent, _ := body.Html()
+		return cleanHTML(htmlContent)
+	}
+	
+	return ""
+}
+
+// cleanHTML performs basic HTML cleaning
+func cleanHTML(htmlStr string) string {
+	// Parse and extract text content while preserving structure
+	doc, err := html.Parse(strings.NewReader(htmlStr))
+	if err != nil {
+		return ""
+	}
+	
+	var texts []string
+	var extractText func(*html.Node)
+	extractText = func(n *html.Node) {
+		if n.Type == html.TextNode {
+			text := strings.TrimSpace(n.Data)
+			if text != "" {
+				texts = append(texts, text)
+			}
+		}
+		for c := n.FirstChild; c != nil; c = c.NextSibling {
+			extractText(c)
+		}
+	}
+	
+	extractText(doc)
+	return strings.Join(texts, "\n")
+}
+
+// isValidURL validates that a URL is safe to fetch
+func isValidURL(urlStr string) error {
+	parsed, err := url.Parse(urlStr)
+	if err != nil {
+		return fmt.Errorf("invalid URL format: %w", err)
+	}
+	
+	// Only allow HTTP/HTTPS
+	if parsed.Scheme != "http" && parsed.Scheme != "https" {
+		return fmt.Errorf("only HTTP/HTTPS URLs allowed")
+	}
+	
+	// Block private IP ranges
+	host := parsed.Hostname()
+	ip := net.ParseIP(host)
+	if ip != nil {
+		if ip.IsPrivate() || ip.IsLoopback() || ip.IsLinkLocalUnicast() || ip.IsMulticast() {
+			return fmt.Errorf("private/internal IP addresses not allowed")
+		}
+	}
+	
+	// Block dangerous hostnames
+	dangerousHosts := []string{"localhost", "127.0.0.1", "0.0.0.0", "::1"}
+	for _, dangerous := range dangerousHosts {
+		if host == dangerous {
+			return fmt.Errorf("local hostnames not allowed")
+		}
+	}
+	
+	return nil
+}
+
+// validateConfig validates the entire configuration
+func validateConfig(config *Config) error {
+	if err := validateOutputPath(config.OutputPath); err != nil {
+		return fmt.Errorf("output path validation failed: %w", err)
+	}
+	
+	if err := isValidURL(config.BaseURL); err != nil {
+		return fmt.Errorf("base URL validation failed: %w", err)
+	}
+	
+	// Limit depth to prevent excessive crawling
+	if config.MaxDepth > 10 {
+		return fmt.Errorf("max depth cannot exceed 10")
+	}
+	
+	// Limit workers to prevent resource exhaustion
+	if config.Workers > 20 {
+		return fmt.Errorf("concurrent workers cannot exceed 20")
+	}
+	
+	// Ensure reasonable timeout values
+	if config.Workers <= 0 {
+		config.Workers = 3 // Default
+	}
+	if config.MaxDepth <= 0 {
+		config.MaxDepth = 2 // Default
+	}
+	
+	return nil
+}

package/pkg/fetcher/html2md.go

@@ -0,0 +1,71 @@
+package fetcher
+
+import (
+	"strings"
+)
+
+// ConvertHTMLToMarkdown converts HTML content to clean markdown
+func ConvertHTMLToMarkdown(htmlContent string) string {
+	if htmlContent == "" {
+		return ""
+	}
+	
+	// Basic HTML to markdown conversion
+	markdownContent := basicHTMLToMarkdown(htmlContent)
+	
+	return strings.TrimSpace(markdownContent)
+}
+
+// basicHTMLToMarkdown provides basic HTML to markdown conversion
+func basicHTMLToMarkdown(html string) string {
+	// Replace common HTML tags with markdown equivalents
+	replacements := map[string]string{
+		"<h1>": "# ",
+		"</h1>": "\n\n",
+		"<h2>": "## ",
+		"</h2>": "\n\n", 
+		"<h3>": "### ",
+		"</h3>": "\n\n",
+		"<h4>": "#### ",
+		"</h4>": "\n\n",
+		"<h5>": "##### ",
+		"</h5>": "\n\n",
+		"<h6>": "###### ",
+		"</h6>": "\n\n",
+		"<p>": "",
+		"</p>": "\n\n",
+		"<br>": "\n",
+		"<br/>": "\n",
+		"<strong>": "**",
+		"</strong>": "**",
+		"<b>": "**",
+		"</b>": "**",
+		"<em>": "*",
+		"</em>": "*",
+		"<i>": "*",
+		"</i>": "*",
+		"<code>": "`",
+		"</code>": "`",
+		"<pre>": "```",
+		"</pre>": "```",
+		"<ul>": "",
+		"</ul>": "\n",
+		"<ol>": "",
+		"</ol>": "\n",
+		"<li>": "- ",
+		"</li>": "\n",
+		"<blockquote>": "> ",
+		"</blockquote>": "\n\n",
+	}
+	
+	result := html
+	for htmlTag, mdReplacement := range replacements {
+		result = strings.ReplaceAll(result, htmlTag, mdReplacement)
+	}
+	
+	// Clean up extra whitespace
+	result = strings.ReplaceAll(result, "\n\n\n", "\n\n")
+	result = strings.TrimSpace(result)
+	
+	return result
+}

package/pkg/fetcher/llmtxt.go

@@ -0,0 +1,36 @@
+package fetcher
+
+import (
+	"bufio"
+	"fmt"
+	"os"
+	"strings"
+)
+
+// GenerateLLMTxt creates an llm.txt file with AI-friendly documentation index
+func GenerateLLMTxt(entries []LLMTxtEntry, outputPath string) error {
+	file, err := os.Create(outputPath)
+	if err != nil {
+		return fmt.Errorf("failed to create llm.txt file: %w", err)
+	}
+	defer file.Close()
+
+	writer := bufio.NewWriter(file)
+	defer writer.Flush()
+
+	// Write header
+	writer.WriteString("# llm.txt - AI-friendly documentation index\n")
+	writer.WriteString("# This file helps LLMs quickly find relevant documentation sections\n\n")
+
+	for _, entry := range entries {
+		// Write entry in the format: [TYPE] Title
+		writer.WriteString(fmt.Sprintf("[%s] %s\n", 
+			strings.ToUpper(entry.Type), entry.Title))
+		// Write URL
+		writer.WriteString(entry.URL + "\n")
+		// Write description
+		writer.WriteString(entry.Description + "\n\n")
+	}
+
+	return nil
+}

package/pkg/fetcher/validator.go

@@ -0,0 +1,109 @@
+package fetcher
+
+import (
+	"fmt"
+	"net"
+	"net/url"
+	"path/filepath"
+	"strings"
+)
+
+// ValidateConfig validates the configuration for security issues
+func ValidateConfig(config *Config) error {
+	if err := validateURL(config.BaseURL); err != nil {
+		return fmt.Errorf("invalid URL: %w", err)
+	}
+	if err := validateOutputPath(config.OutputPath); err != nil {
+		return fmt.Errorf("invalid output path: %w", err)
+	}
+	if config.MaxDepth > 10 {
+		return fmt.Errorf("max depth too high (maximum allowed: 10)")
+	}
+	if config.Workers > 20 {
+		return fmt.Errorf("too many concurrent workers (maximum allowed: 20)")
+	}
+	return nil
+}
+
+// validateURL checks if the URL is safe to fetch
+func validateURL(urlStr string) error {
+	parsed, err := url.Parse(urlStr)
+	if err != nil {
+		return err
+	}
+
+	// Only allow HTTP and HTTPS
+	if parsed.Scheme != "http" && parsed.Scheme != "https" {
+		return fmt.Errorf("only HTTP and HTTPS URLs are allowed")
+	}
+
+	// Block private IP ranges and localhost
+	host := parsed.Hostname()
+	ip := net.ParseIP(host)
+	if ip != nil {
+		if ip.IsPrivate() || ip.IsLoopback() || ip.IsLinkLocalUnicast() || ip.IsMulticast() {
+			return fmt.Errorf("access to private/internal IP addresses is not allowed")
+		}
+	}
+
+	// Block dangerous hostnames
+	dangerousHosts := []string{"localhost", "127.0.0.1", "0.0.0.0", "::1"}
+	for _, dangerous := range dangerousHosts {
+		if strings.ToLower(host) == dangerous {
+			return fmt.Errorf("access to localhost is not allowed")
+		}
+	}
+
+	return nil
+}
+
+// validateOutputPath ensures the output path is safe
+func validateOutputPath(path string) error {
+	// Don't allow absolute paths that start with /
+	if strings.HasPrefix(path, "/") {
+		return fmt.Errorf("absolute paths are not allowed")
+	}
+
+	// Don't allow paths that contain ..
+	if strings.Contains(path, "..") {
+		return fmt.Errorf("relative path traversal (..) is not allowed")
+	}
+
+	// Don't allow paths that contain ~
+	if strings.Contains(path, "~") {
+		return fmt.Errorf("home directory expansion (~) is not allowed")
+	}
+
+	// Resolve to absolute path to check final destination
+	absPath, err := filepath.Abs(path)
+	if err != nil {
+		return err
+	}
+
+	// Get current working directory
+	cwd, err := filepath.Abs(".")
+	if err != nil {
+		return err
+	}
+
+	// Ensure the absolute path is within the current working directory
+	if !strings.HasPrefix(absPath, cwd) {
+		return fmt.Errorf("output path must be within the current working directory")
+	}
+
+	// Check file extension - only allow safe extensions
+	allowedExtensions := []string{".md", ".txt", ".llm.txt"}
+	ext := filepath.Ext(path)
+	isAllowed := false
+	for _, allowed := range allowedExtensions {
+		if ext == allowed {
+			isAllowed = true
+			break
+		}
+	}
+	if !isAllowed {
+		return fmt.Errorf("only .md, .txt, and .llm.txt file extensions are allowed")
+	}
+
+	return nil
+}

package/pkg/fetcher/writer.go

@@ -0,0 +1,32 @@
+package fetcher
+
+import (
+	"bufio"
+	"os"
+	"strings"
+)
+
+// writeResults writes the fetched documentation to the output file
+func writeResults(outputPath string, resultsChan <-chan string) error {
+	file, err := os.Create(outputPath)
+	if err != nil {
+		return err
+	}
+	defer file.Close()
+	
+	writer := bufio.NewWriter(file)
+	defer writer.Flush()
+	
+	// Write header
+	header := "# Documentation\n\nThis file contains documentation fetched by DocFetch.\n\n---\n\n"
+	writer.WriteString(header)
+	
+	// Write all results
+	for result := range resultsChan {
+		if strings.TrimSpace(result) != "" {
+			writer.WriteString(result)
+		}
+	}
+	
+	return nil
+}

package/pyproject.toml

@@ -0,0 +1,37 @@
+[build-system]
+requires = ["setuptools>=45", "wheel", "setuptools_scm[toml]>=6.2"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "doc-fetch"
+version = "1.0.1"
+description = "Dynamic documentation fetching CLI that converts entire documentation sites to single markdown files for AI/LLM consumption"
+readme = "README.md"
+authors = [{name = "AlphaTechini", email = "rehobothokoibu@gmail.com"}]
+license = {text = "MIT"}
+classifiers = [
+    "Development Status :: 5 - Production/Stable",
+    "Intended Audience :: Developers",
+    "License :: OSI Approved :: MIT License",
+    "Operating System :: OS Independent",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.8",
+    "Programming Language :: Python :: 3.9",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+    "Topic :: Documentation",
+    "Topic :: Software Development :: Documentation",
+    "Topic :: Utilities",
+]
+keywords = ["documentation", "ai", "llm", "markdown", "crawler", "security"]
+requires-python = ">=3.8"
+dependencies = []
+
+[project.urls]
+Homepage = "https://github.com/AlphaTechini/doc-fetch"
+Repository = "https://github.com/AlphaTechini/doc-fetch"
+Documentation = "https://github.com/AlphaTechini/doc-fetch#readme"
+
+[project.scripts]
+doc-fetch = "doc_fetch.cli:main"