doc-fetch-cli 1.0.2 → 1.1.1

package/CONTRIBUTING.md

@@ -0,0 +1,274 @@
+# Contributing to DocFetch
+
+Thank you for your interest in contributing to DocFetch! 🎉 This guide will help you get started.
+
+## 📋 Quick Overview
+
+1. **Create an issue first** - Always start by opening an issue to discuss your change
+2. **Wait for feedback** - Maintainers will respond and provide guidance
+3. **Fork and develop** - Once approved, fork the repo and make your changes
+4. **Submit a PR** - Open a pull request referencing the issue
+5. **Review process** - Maintainers will review and provide feedback
+6. **Merge** - Once approved, your contribution will be merged!
+
+---
+
+## 🐛 Before You Start: Create an Issue
+
+**⚠️ IMPORTANT: Always create an issue before submitting a PR!**
+
+This helps us:
+- Avoid duplicate work
+- Discuss the best approach
+- Ensure your contribution aligns with project goals
+- Get early feedback from maintainers
+
+### Types of Issues
+
+#### 🐞 Bug Reports
+Include:
+- Clear description of the bug
+- Steps to reproduce
+- Expected vs actual behavior
+- Environment details (OS, Go version, DocFetch version)
+- Sample command that triggers the bug
+- Error messages or logs
+
+#### ✨ Feature Requests
+Include:
+- Clear description of the feature
+- Use case / problem it solves
+- Example usage
+- Any relevant links or references
+
+#### 📝 Documentation Improvements
+Include:
+- What needs improvement
+- Why it's needed
+- Suggested changes
+
+#### ⚡ Performance Improvements
+Include:
+- Current performance metrics
+- Proposed improvements
+- Benchmark results (if available)
+
+---
+
+## 🚀 Development Setup
+
+### Prerequisites
+
+- Go 1.21 or later
+- Git
+- Make (optional, for running tests)
+
+### Fork and Clone
+
+```bash
+# Fork the repository on GitHub, then:
+git clone https://github.com/YOUR_USERNAME/doc-fetch.git
+cd doc-fetch
+
+# Add upstream remote
+git remote add upstream https://github.com/AlphaTechini/doc-fetch.git
+```
+
+### Build from Source
+
+```bash
+# Build the binary
+go build -o doc-fetch ./cmd/docfetch
+
+# Test it works
+./doc-fetch --help
+```
+
+### Run Tests
+
+```bash
+# Run all tests
+go test ./...
+
+# Run tests with coverage
+go test -cover ./...
+
+# Run specific package tests
+go test ./pkg/fetcher/...
+```
+
+---
+
+## 💻 Making Changes
+
+### Branch Naming
+
+Use descriptive branch names:
+- `fix/content-extraction-bug`
+- `feat/add-pdf-support`
+- `docs/update-readme-examples`
+- `perf/improve-concurrent-fetching`
+
+### Code Style
+
+Follow Go best practices:
+- Run `go fmt` before committing
+- Run `go vet` to catch issues
+- Write clear, concise comments
+- Keep functions small and focused
+- Use meaningful variable names
+
+### Testing Requirements
+
+- Add tests for new features
+- Ensure existing tests pass
+- Include edge cases
+- Test with real documentation sites
+
+Example test:
+```go
+func TestContentExtraction(t *testing.T) {
+    doc := createTestDocument()
+    content := cleanContent(doc)
+    
+    if len(content) == 0 {
+        t.Error("Expected content to be extracted")
+    }
+    
+    if !strings.Contains(content, "expected text") {
+        t.Error("Expected content to contain specific text")
+    }
+}
+```
+
+---
+
+## 📤 Submitting a Pull Request
+
+### PR Checklist
+
+Before submitting your PR, ensure:
+
+- [ ] You created an issue first and referenced it in the PR
+- [ ] Your code follows Go style guidelines
+- [ ] All tests pass (`go test ./...`)
+- [ ] You've added tests for new functionality
+- [ ] You've updated documentation if needed
+- [ ] Your commit messages are clear and descriptive
+- [ ] You've rebased on the latest main branch
+
+### PR Template
+
+When creating your PR, include:
+
+```markdown
+## Description
+Brief description of changes
+
+## Related Issue
+Fixes #123 (or "Related to #123")
+
+## Type of Change
+- [ ] Bug fix
+- [ ] New feature
+- [ ] Breaking change
+- [ ] Documentation update
+- [ ] Performance improvement
+- [ ] Refactoring
+
+## Testing
+Describe how you tested this:
+- [ ] Unit tests added/updated
+- [ ] Manual testing with real docs
+- [ ] Tested on: [list platforms]
+
+## Example Usage
+Show example command and output if applicable
+
+## Checklist
+- [ ] Code follows project guidelines
+- [ ] Self-review completed
+- [ ] Comments added where needed
+- [ ] Tests pass locally
+```
+
+---
+
+## 🔍 Review Process
+
+1. **Automated Checks**: CI runs tests and linting
+2. **Maintainer Review**: At least one maintainer reviews
+3. **Feedback**: You may be asked to make changes
+4. **Approval**: Once approved, PR is merged
+5. **Release**: Changes included in next release
+
+Typical timeline: 3-7 days for review
+
+---
+
+## 📖 Contribution Ideas
+
+Looking for ways to contribute? Here are some ideas:
+
+### Easy Wins
+- Fix typos in documentation
+- Add more examples to README
+- Improve error messages
+- Add unit tests for existing code
+
+### Intermediate
+- Add support for new documentation site formats
+- Improve content extraction selectors
+- Add progress indicators
+- Enhance LLM.txt generation
+
+### Advanced
+- Add PDF export support
+- Implement incremental updates
+- Add authentication support for private docs
+- Create plugin system for custom extractors
+
+---
+
+## 🤝 Community Guidelines
+
+### Be Respectful
+- Treat everyone with respect
+- Welcome newcomers
+- Provide constructive feedback
+- Assume good intentions
+
+### Communication
+- Use clear, concise language
+- Explain your reasoning
+- Ask questions if unsure
+- Respond to feedback promptly
+
+### Collaboration
+- Work with maintainers, not against them
+- Be open to suggestions
+- Help other contributors
+- Share knowledge
+
+---
+
+## 📜 License
+
+By contributing to DocFetch, you agree that your contributions will be licensed under the MIT License.
+
+---
+
+## ❓ Questions?
+
+- **General questions**: Open a discussion on GitHub
+- **Bug reports**: Create an issue
+- **Feature requests**: Create an issue
+- **Quick questions**: Check existing issues/discussions first
+
+---
+
+## 🙏 Thank You!
+
+Your contributions make DocFetch better for everyone. Whether it's a typo fix, a new feature, or better documentation - we appreciate your time and effort!
+
+Happy coding! 🚀

package/bin/postinstall.js

@@ -0,0 +1,83 @@
+#!/usr/bin/env node
+/**
+ * Post-install script for doc-fetch-cli
+ * Checks if global bin directory is in PATH and provides helpful instructions
+ */
+
+const { execSync } = require('child_process');
+const path = require('path');
+const os = require('os');
+const fs = require('fs');
+
+console.log('🎉 DocFetch CLI installed successfully!\n');
+
+// Get npm global prefix
+let globalPrefix;
+try {
+    globalPrefix = execSync('npm config get prefix', { encoding: 'utf8' }).trim();
+} catch (error) {
+    console.error('⚠️  Could not determine npm global prefix');
+    globalPrefix = null;
+}
+
+if (globalPrefix) {
+    const binDir = path.join(globalPrefix, 'bin');
+    const isWindows = os.platform() === 'win32';
+    
+    console.log(`📦 Installed to: ${binDir}\n`);
+    
+    // Check if bin directory is in PATH
+    const pathEnv = process.env.PATH || '';
+    const pathDirs = pathEnv.split(isWindows ? ';' : ':');
+    const isInPath = pathDirs.some(dir => path.resolve(dir) === path.resolve(binDir));
+    
+    if (!isInPath) {
+        console.log('⚠️  WARNING: Global bin directory is not in your PATH!\n');
+        console.log('To use doc-fetch-cli, add this directory to your PATH:\n');
+        console.log(`   ${binDir}\n`);
+        
+        // Provide platform-specific instructions
+        const shell = process.env.SHELL || '/bin/bash';
+        const isZsh = shell.includes('zsh');
+        const isBash = shell.includes('bash');
+        
+        console.log('Quick fix:\n');
+        
+        if (isWindows) {
+            console.log('1. Open System Properties → Environment Variables');
+            console.log('2. Edit PATH variable');
+            console.log('3. Add this path:');
+            console.log(`   ${binDir}`);
+            console.log('4. Restart your terminal\n');
+        } else if (isZsh) {
+            console.log('Add this to your ~/.zshrc:');
+            console.log(`   export PATH="${binDir}:$PATH"\n`);
+            console.log('Then run: source ~/.zshrc\n');
+        } else if (isBash) {
+            console.log('Add this to your ~/.bashrc or ~/.bash_profile:');
+            console.log(`   export PATH="${binDir}:$PATH"\n`);
+            console.log('Then run: source ~/.bashrc\n');
+        }
+        
+        console.log('Alternative: Use npx without installing globally\n');
+        console.log('   npx doc-fetch-cli --url https://docs.example.com --output docs.md\n');
+    } else {
+        console.log('✅ Global bin directory is in your PATH\n');
+        console.log('You can now use doc-fetch-cli!\n');
+        console.log('Example usage:');
+        console.log('   doc-fetch --url https://docs.python.org/3 --output docs.md --llm-txt\n');
+        
+        // Test if the command works
+        try {
+            execSync('doc-fetch --version', { encoding: 'utf8', stdio: 'pipe' });
+            console.log('✅ Command verified working!\n');
+        } catch (error) {
+            console.log('⚠️  Command not found in current shell session.\n');
+            console.log('Try running: hash -r  (to clear command cache)\n');
+            console.log('Or restart your terminal.\n');
+        }
+    }
+}
+
+console.log('📚 Documentation: https://github.com/AlphaTechini/doc-fetch\n');
+console.log('✨ Pro tip: Use --llm-txt flag to generate AI-friendly index files!\n');

package/cmd/docfetch/main.go

@@ -36,7 +36,8 @@ func main() {
 		log.Fatalf("Configuration error: %v", err)
 	}
 
-	err := fetcher.Run(config)
+	// Use optimized high-performance fetcher
+	err := fetcher.RunOptimized(config)
 	if err != nil {
 		log.Fatalf("Failed to fetch documentation: %v", err)
 	}

package/doc_fetch.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: doc-fetch
-Version: 1.0.1
+Version: 1.1.1
 Summary: Dynamic documentation fetching CLI that converts entire documentation sites to single markdown files for AI/LLM consumption
 Home-page: https://github.com/AlphaTechini/doc-fetch
 Author: AlphaTechini

package/doc_fetch.egg-info/SOURCES.txt

@@ -1,7 +1,18 @@
+CONTRIBUTING.md
 README.md
+SECURITY.md
+doc-fetch
+doc-fetch_darwin_amd64
+doc-fetch_linux_amd64
+doc-fetch_windows_amd64.exe
 go.mod
+go.sum
+package.json
 pyproject.toml
 setup.py
+bin/doc-fetch.js
+bin/install.js
+bin/postinstall.js
 cmd/docfetch/main.go
 doc_fetch/__init__.py
 doc_fetch/__main__.py
@@ -14,6 +25,12 @@ doc_fetch.egg-info/not-zip-safe
 doc_fetch.egg-info/top_level.txt
 docs/usage.md
 examples/golang-example.sh
+pkg/fetcher/classifier.go
+pkg/fetcher/describer.go
+pkg/fetcher/extract_nav.go
 pkg/fetcher/fetcher.go
+pkg/fetcher/fetcher_optimized.go
 pkg/fetcher/html2md.go
+pkg/fetcher/llmtxt.go
+pkg/fetcher/validator.go
 pkg/fetcher/writer.go

package/package.json

@@ -1,12 +1,12 @@
 {
   "name": "doc-fetch-cli",
-  "version": "1.0.2",
+  "version": "1.1.1",
   "description": "Dynamic documentation fetching CLI that converts entire documentation sites to single markdown files for AI/LLM consumption",
   "bin": {
     "doc-fetch": "./bin/doc-fetch.js"
   },
   "scripts": {
-    "postinstall": "node ./bin/install.js"
+    "postinstall": "node ./bin/postinstall.js"
   },
   "repository": {
     "type": "git",

package/pkg/fetcher/extract_nav.go

@@ -0,0 +1,163 @@
+package fetcher
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/PuerkitoBio/goquery"
+)
+
+// ExtractNavigationStructure extracts nav elements with h2/h3, ul/li, and hrefs
+func ExtractNavigationStructure(doc *goquery.Document) string {
+	var result strings.Builder
+	
+	result.WriteString("# Navigation Structure\n\n")
+	
+	// Find all nav elements
+	doc.Find("nav").Each(func(i int, nav *goquery.Selection) {
+		result.WriteString(fmt.Sprintf("## Navigation Block %d\n\n", i+1))
+		
+		// Look for headings in nav
+		nav.Find("h1, h2, h3, h4, h5, h6").Each(func(j int, h *goquery.Selection) {
+			tagName := h.Get(0).Data
+			text := strings.TrimSpace(h.Text())
+			if text != "" {
+				result.WriteString(fmt.Sprintf("### %s: %s\n\n", tagName, text))
+			}
+			
+			// Find ul under this heading
+			h.NextFiltered("ul").Each(func(k int, ul *goquery.Selection) {
+				result.WriteString(extractListWithLinks(ul, 1))
+			})
+		})
+		
+		// Also find ul directly in nav
+		nav.ChildrenFiltered("ul").Each(func(k int, ul *goquery.Selection) {
+			result.WriteString(extractListWithLinks(ul, 1))
+		})
+		
+		result.WriteString("---\n\n")
+	})
+	
+	// Also look for elements with navigation-related classes/ids
+	navSelectors := []string{
+		"[class*='nav']",
+		"[id*='nav']",
+		"[class*='menu']",
+		"[id*='menu']",
+		"[role='navigation']",
+		".toc",
+		"#toc",
+		"[class*='toc']",
+		"[id*='toc']",
+	}
+	
+	for _, selector := range navSelectors {
+		doc.Find(selector).Each(func(i int, s *goquery.Selection) {
+			// Skip if already processed as nav element
+			if s.Parent().Is("nav") {
+				return
+			}
+			
+			result.WriteString(fmt.Sprintf("## Navigation Element (matched: %s)\n\n", selector))
+			
+			// Extract headings
+			s.Find("h1, h2, h3, h4, h5, h6").Each(func(j int, h *goquery.Selection) {
+				tagName := h.Get(0).Data
+				text := strings.TrimSpace(h.Text())
+				if text != "" {
+					result.WriteString(fmt.Sprintf("### %s: %s\n\n", tagName, text))
+				}
+			})
+			
+			// Extract lists with links
+			s.Find("ul, ol").Each(func(k int, list *goquery.Selection) {
+				result.WriteString(extractListWithLinks(list, 1))
+			})
+			
+			result.WriteString("---\n\n")
+		})
+	}
+	
+	return result.String()
+}
+
+// extractListWithLinks extracts list items with their href attributes
+func extractListWithLinks(list *goquery.Selection, indentLevel int) string {
+	var result strings.Builder
+	
+	indent := strings.Repeat("  ", indentLevel)
+	
+	list.Find("> li").Each(func(i int, li *goquery.Selection) {
+		// Get the text
+		text := strings.TrimSpace(li.Text())
+		
+		// Find any links in this li
+		li.Find("a[href]").Each(func(j int, a *goquery.Selection) {
+			href, exists := a.Attr("href")
+			linkText := strings.TrimSpace(a.Text())
+			if exists && href != "" {
+				result.WriteString(fmt.Sprintf("%s- [%s](%s)\n", indent, linkText, href))
+			}
+		})
+		
+		// If no links found, just add the text
+		if li.Find("a[href]").Length() == 0 && text != "" {
+			result.WriteString(fmt.Sprintf("%s- %s\n", indent, text))
+		}
+		
+		// Recursively process nested lists
+		li.ChildrenFiltered("ul, ol").Each(func(k int, nested *goquery.Selection) {
+			result.WriteString(extractListWithLinks(nested, indentLevel+1))
+		})
+	})
+	
+	return result.String()
+}
+
+// ExtractAllLinks extracts all links from the page with context
+func ExtractAllLinks(doc *goquery.Document, baseURL string) string {
+	var result strings.Builder
+	
+	result.WriteString("# All Links Found\n\n")
+	
+	linksFound := 0
+	
+	// Group links by section
+	doc.Find("section, article, div[class*='content'], div[id*='content']").Each(func(i int, section *goquery.Selection) {
+		sectionLinks := 0
+		var sectionResult strings.Builder
+		
+		// Get section title
+		title := ""
+		section.Find("h1, h2, h3").First().Each(func(j int, h *goquery.Selection) {
+			title = strings.TrimSpace(h.Text())
+		})
+		
+		if title == "" {
+			title = fmt.Sprintf("Section %d", i+1)
+		}
+		
+		sectionResult.WriteString(fmt.Sprintf("## %s\n\n", title))
+		
+		// Find all links in this section
+		section.Find("a[href]").Each(func(j int, a *goquery.Selection) {
+			href, exists := a.Attr("href")
+			text := strings.TrimSpace(a.Text())
+			if exists && href != "" && text != "" {
+				sectionResult.WriteString(fmt.Sprintf("- [%s](%s)\n", text, href))
+				sectionLinks++
+				linksFound++
+			}
+		})
+		
+		if sectionLinks > 0 {
+			result.WriteString(sectionResult.String())
+			result.WriteString("\n")
+		}
+	})
+	
+	result.WriteString(fmt.Sprintf("\n**Total links found: %d**\n", linksFound))
+	
+	return result.String()
+}

package/pkg/fetcher/fetcher.go

@@ -195,53 +195,136 @@ func worker(config Config, pagesChan <-chan *Page, resultsChan chan<- string, mu
 	}
 }
 
-// cleanContent extracts and cleans the main documentation content
+// cleanContent extracts and cleans the main documentation content using multiple strategies
 func cleanContent(doc *goquery.Document) string {
-	// Common selectors for documentation content
-	selectors := []string{
+	// Strategy 1: Try semantic HTML5 elements (most reliable)
+	semanticSelectors := []string{
 		"main",
 		"article",
+		"[role='main']",
+		"[role='article']",
+	}
+	
+	for _, selector := range semanticSelectors {
+		if el := doc.Find(selector); el.Length() > 0 {
+			content := extractTextContent(el)
+			if len(content) > 200 { // Minimum viable content
+				return content
+			}
+		}
+	}
+	
+	// Strategy 2: Try common class/id patterns
+	classSelectors := []string{
 		".content",
-		".docs-content",
+		".docs-content", 
 		"#main-content",
 		".documentation",
 		".post-content",
 		".markdown-body",
 		".content-wrapper",
 		".doc-content",
+		".document",
+		".entry-content",
+		".page-content",
+		".article-content",
+		"[class*='content']",
+		"[class*='docs']",
+		"[class*='document']",
+		"[id*='content']",
+		"[id*='main']",
 	}
 	
-	// Try each selector
-	for _, selector := range selectors {
+	for _, selector := range classSelectors {
 		if el := doc.Find(selector); el.Length() > 0 {
-			// Remove unwanted elements
-			el.Find("nav, header, footer, .sidebar, .toc, .navigation, script, style, .ad, .advertisement").Remove()
-			
-			// Convert to HTML and then clean
-			htmlContent, err := el.Html()
-			if err != nil {
-				continue
+			content := extractTextContent(el)
+			if len(content) > 200 {
+				return content
 			}
+		}
+	}
+	
+	// Strategy 3: Look for sections with high text density
+	var bestSection *goquery.Selection
+	maxTextLen := 0
+	
+	doc.Find("section, div").Each(func(i int, s *goquery.Selection) {
+		text := strings.TrimSpace(s.Text())
+		if len(text) > maxTextLen {
+			// Check if this section has more text than child elements
+			childText := 0
+			s.Children().Each(func(j int, c *goquery.Selection) {
+				childText += len(strings.TrimSpace(c.Text()))
+			})
 			
-			// Basic HTML cleaning
-			cleaned := cleanHTML(htmlContent)
-			if cleaned != "" {
-				return cleaned
+			// If parent has significantly more text, it's likely the main content
+			if len(text) > childText + (childText/2) && len(text) > 500 {
+				maxTextLen = len(text)
+				bestSection = s
 			}
 		}
+	})
+	
+	if bestSection != nil {
+		content := extractTextContent(bestSection)
+		if len(content) > 200 {
+			return content
+		}
 	}
 	
-	// Fallback: try to get body content
+	// Strategy 4: Fallback to body with aggressive cleaning
 	body := doc.Find("body")
 	if body.Length() > 0 {
-		body.Find("nav, header, footer, .sidebar, .toc, .navigation, script, style, .ad, .advertisement").Remove()
+		// Remove all non-content elements aggressively
+		body.Find("nav, header, footer, aside, script, style, form, iframe, .sidebar, .toc, .navigation, .menu, .ads, .advertisement, [class*='nav'], [class*='menu'], [class*='sidebar'], [class*='footer'], [class*='header']").Remove()
+		
+		// Find the largest remaining container
+		var largest *goquery.Selection
+		largestSize := 0
+		
+		body.Find("*").Each(func(i int, s *goquery.Selection) {
+			text := strings.TrimSpace(s.Text())
+			if len(text) > largestSize && s.Children().Length() < 50 {
+				largestSize = len(text)
+				largest = s
+			}
+		})
+		
+		if largest != nil {
+			content := extractTextContent(largest)
+			if len(content) > 200 {
+				return content
+			}
+		}
+		
+		// Last resort: entire body
 		htmlContent, _ := body.Html()
-		return cleanHTML(htmlContent)
+		cleaned := cleanHTML(htmlContent)
+		if len(cleaned) > 200 {
+			return cleaned
+		}
 	}
 	
 	return ""
 }
 
+// extractTextContent extracts and cleans text from a selection
+func extractTextContent(sel *goquery.Selection) string {
+	// Clone the selection to avoid modifying original
+	clone := sel.Clone()
+	
+	// Remove unwanted elements
+	clone.Find("nav, header, footer, aside, script, style, form, iframe, .sidebar, .toc, .navigation, .menu, .ads, .advertisement, button, [class*='nav'], [class*='menu'], [class*='sidebar'], [class*='footer'], [class*='header'], [class*='button'], [onclick], [role='navigation'], [role='banner'], [role='contentinfo']").Remove()
+	
+	// Get HTML and convert to clean text
+	htmlContent, err := clone.Html()
+	if err != nil {
+		return ""
+	}
+	
+	return cleanHTML(htmlContent)
+}
+
 // cleanHTML performs basic HTML cleaning
 func cleanHTML(htmlStr string) string {
 	// Parse and extract text content while preserving structure

package/pkg/fetcher/fetcher_optimized.go

@@ -0,0 +1,318 @@
+package fetcher
+
+import (
+	"bufio"
+	"context"
+	"fmt"
+	"log"
+	"net"
+	"net/http"
+	"net/url"
+	"os"
+	"strings"
+	"sync"
+	"sync/atomic"
+	"time"
+
+	"github.com/PuerkitoBio/goquery"
+)
+
+// OptimizedFetcher uses advanced Go concurrency patterns for 10x speedup
+type OptimizedFetcher struct {
+	config        Config
+	httpClient    *http.Client
+	urlQueue      chan string
+	visited       sync.Map // Concurrent map instead of mutex-protected map
+	resultsChan   chan string
+	llmEntries    []LLMTxtEntry
+	llmMutex      sync.Mutex
+	pageCount     int32
+	errorCount    int32
+	ctx           context.Context
+	cancel        context.CancelFunc
+}
+
+// RunOptimized executes documentation fetching with maximum concurrency
+func RunOptimized(config Config) error {
+	if err := validateConfig(&config); err != nil {
+		return fmt.Errorf("invalid configuration: %w", err)
+	}
+
+	log.Printf("🚀 Starting HIGH-PERFORMANCE documentation fetch from: %s", config.BaseURL)
+	log.Printf("   Workers: %d | Max Depth: %d | Concurrency: Enabled", config.Workers, config.MaxDepth)
+
+	fetcher := &OptimizedFetcher{
+		config:      config,
+		urlQueue:    make(chan string, config.Workers*100), // Large buffer for URLs
+		resultsChan: make(chan string, config.Workers*10), // Larger buffer
+		httpClient: createOptimizedHTTPClient(config.Workers),
+	}
+
+	fetcher.ctx, fetcher.cancel = context.WithTimeout(context.Background(), 10*time.Minute)
+	defer fetcher.cancel()
+
+	startTime := time.Now()
+	
+	// Start result writer in background
+	var writeWg sync.WaitGroup
+	writeWg.Add(1)
+	go func() {
+		defer writeWg.Add(-1)
+		writeResultsOptimized(config.OutputPath, fetcher.resultsChan)
+	}()
+
+	// Start worker pool
+	var workerWg sync.WaitGroup
+	for i := 0; i < config.Workers; i++ {
+		workerWg.Add(1)
+		go fetcher.worker(i, &workerWg)
+	}
+
+	// Submit initial URL
+	fetcher.submitPage(config.BaseURL, 0)
+
+	// Close URL queue when all pages are processed
+	go func() {
+		workerWg.Wait()
+		close(fetcher.urlQueue)
+	}()
+
+	// Wait for all workers to complete
+	workerWg.Wait()
+	close(fetcher.resultsChan)
+
+	// Wait for results to be written
+	writeWg.Wait()
+
+	elapsed := time.Since(startTime)
+	pagesFetched := atomic.LoadInt32(&fetcher.pageCount)
+	errors := atomic.LoadInt32(&fetcher.errorCount)
+
+	log.Printf("✅ Fetch completed!")
+	log.Printf("   📊 Pages fetched: %d", pagesFetched)
+	log.Printf("   ⏱️  Time elapsed: %v", elapsed)
+	log.Printf("   📈 Speed: %.2f pages/second", float64(pagesFetched)/elapsed.Seconds())
+	log.Printf("   ❌ Errors: %d", errors)
+
+	// Generate LLM.txt if requested
+	if config.GenerateLLMTxt && len(fetcher.llmEntries) > 0 {
+		llmTxtPath := strings.TrimSuffix(config.OutputPath, ".md") + ".llm.txt"
+		if err := GenerateLLMTxt(fetcher.llmEntries, llmTxtPath); err != nil {
+			log.Printf("⚠️  Warning: Failed to generate llm.txt: %v", err)
+		} else {
+			log.Printf("📝 LLM.txt generated: %s (%d entries)", llmTxtPath, len(fetcher.llmEntries))
+		}
+	}
+
+	return nil
+}
+
+// createOptimizedHTTPClient creates a high-performance HTTP client with connection pooling
+func createOptimizedHTTPClient(workers int) *http.Client {
+	return &http.Client{
+		Timeout: 30 * time.Second,
+		Transport: &http.Transport{
+			MaxIdleConns:        workers * 2,
+			MaxIdleConnsPerHost: workers,
+			IdleConnTimeout:     90 * time.Second,
+			DisableCompression:  false,
+			DisableKeepAlives:   false,
+			DialContext: (&net.Dialer{
+				Timeout:   10 * time.Second,
+				KeepAlive: 30 * time.Second,
+			}).DialContext,
+			TLSHandshakeTimeout: 10 * time.Second,
+		},
+	}
+}
+
+// worker processes URLs from the submission queue
+func (f *OptimizedFetcher) worker(id int, wg *sync.WaitGroup) {
+	defer wg.Done()
+	
+	for url := range f.urlQueue {
+		select {
+		case <-f.ctx.Done():
+			return
+		default:
+			f.processURL(url, 0)
+		}
+	}
+}
+
+// submitPage adds a URL to be fetched (with depth tracking)
+func (f *OptimizedFetcher) submitPage(pageURL string, depth int) {
+	if depth > f.config.MaxDepth {
+		return
+	}
+
+	// Check if already visited using atomic operation
+	if _, loaded := f.visited.LoadOrStore(pageURL, true); loaded {
+		return
+	}
+
+	select {
+	case f.urlQueue <- pageURL:
+		// Successfully queued
+	default:
+		// Queue full, skip this URL
+		log.Printf("⚠️  Queue full, skipping: %s", pageURL)
+	}
+}
+
+// processURL fetches and processes a single URL
+func (f *OptimizedFetcher) processURL(pageURL string, depth int) {
+	atomic.AddInt32(&f.pageCount, 1)
+
+	startTime := time.Now()
+	
+	// Validate URL
+	if err := isValidURL(pageURL); err != nil {
+		atomic.AddInt32(&f.errorCount, 1)
+		log.Printf("❌ Invalid URL %s: %v", pageURL, err)
+		return
+	}
+
+	// Fetch the page
+	resp, err := f.httpClient.Get(pageURL)
+	if err != nil {
+		atomic.AddInt32(&f.errorCount, 1)
+		log.Printf("❌ Error fetching %s: %v", pageURL, err)
+		return
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != 200 {
+		atomic.AddInt32(&f.errorCount, 1)
+		log.Printf("❌ Non-200 status %d for %s", resp.StatusCode, pageURL)
+		return
+	}
+
+	// Parse HTML concurrently
+	doc, err := goquery.NewDocumentFromReader(resp.Body)
+	if err != nil {
+		atomic.AddInt32(&f.errorCount, 1)
+		log.Printf("❌ Error parsing HTML for %s: %v", pageURL, err)
+		return
+	}
+
+	// Extract content
+	content := cleanContent(doc)
+	if content == "" {
+		atomic.AddInt32(&f.errorCount, 1)
+		log.Printf("⚠️  No content found for %s", pageURL)
+		return
+	}
+
+	// Extract title
+	title := doc.Find("title").Text()
+	if title == "" {
+		title = pageURL
+	}
+
+	// Send result
+	f.resultsChan <- fmt.Sprintf("## %s\n\n%s\n\n---\n\n", title, content)
+
+	// Generate LLM.txt entry if requested
+	if f.config.GenerateLLMTxt {
+		cleanTitle := CleanTitle(title)
+		entryType := ClassifyPage(pageURL, cleanTitle)
+		description := ExtractDescription(content)
+
+		entry := LLMTxtEntry{
+			Type:        entryType,
+			Title:       cleanTitle,
+			URL:         pageURL,
+			Description: description,
+		}
+
+		f.llmMutex.Lock()
+		f.llmEntries = append(f.llmEntries, entry)
+		f.llmMutex.Unlock()
+	}
+
+	// Extract links for crawling (if depth allows)
+	if depth < f.config.MaxDepth {
+		f.extractAndSubmitLinks(doc, pageURL, depth+1)
+	}
+
+	elapsed := time.Since(startTime)
+	log.Printf("✅ Fetched %s (%.2fs)", pageURL, elapsed.Seconds())
+}
+
+// extractAndSubmitLinks finds and queues all internal links
+func (f *OptimizedFetcher) extractAndSubmitLinks(doc *goquery.Document, baseURL string, depth int) {
+	base, err := url.Parse(baseURL)
+	if err != nil {
+		return
+	}
+
+	doc.Find("a[href]").Each(func(i int, s *goquery.Selection) {
+		href, exists := s.Attr("href")
+		if !exists {
+			return
+		}
+
+		// Resolve relative URLs
+		resolvedURL, err := base.Parse(href)
+		if err != nil {
+			return
+		}
+
+		// Only follow same-domain links
+		if resolvedURL.Host != "" && resolvedURL.Host != base.Host {
+			return
+		}
+
+		// Skip non-HTML resources
+		if isNonHTMLResource(resolvedURL.Path) {
+			return
+		}
+
+		f.submitPage(resolvedURL.String(), depth)
+	})
+}
+
+// isNonHTMLResource checks if URL points to non-HTML resources
+func isNonHTMLResource(path string) bool {
+	extensions := []string{".pdf", ".zip", ".tar", ".gz", ".exe", ".dmg", ".pkg", ".deb", ".rpm"}
+	pathLower := strings.ToLower(path)
+	
+	for _, ext := range extensions {
+		if strings.HasSuffix(pathLower, ext) {
+			return true
+		}
+	}
+	return false
+}
+
+// writeResultsOptimized writes results to file efficiently
+func writeResultsOptimized(outputPath string, resultsChan <-chan string) error {
+	file, err := os.Create(outputPath)
+	if err != nil {
+		return err
+	}
+	defer file.Close()
+
+	writer := bufio.NewWriterSize(file, 32*1024) // 32KB buffer for better I/O
+	defer writer.Flush()
+
+	// Write header
+	header := "# Documentation\n\nThis file contains documentation fetched by DocFetch.\n\n---\n\n"
+	writer.WriteString(header)
+
+	count := 0
+	for result := range resultsChan {
+		if strings.TrimSpace(result) != "" {
+			writer.WriteString(result)
+			count++
+			
+			// Flush periodically to avoid memory buildup
+			if count%10 == 0 {
+				writer.Flush()
+			}
+		}
+	}
+
+	return nil
+}

package/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "doc-fetch"
-version = "1.0.1"
+version = "1.1.1"
 description = "Dynamic documentation fetching CLI that converts entire documentation sites to single markdown files for AI/LLM consumption"
 readme = "README.md"
 authors = [{name = "AlphaTechini", email = "rehobothokoibu@gmail.com"}]

package/setup.py

@@ -118,7 +118,7 @@ with open("README.md", "r", encoding="utf-8") as fh:
 
 setup(
     name="doc-fetch",
-    version="1.0.1",
+    version="1.1.1",
     author="AlphaTechini",
     author_email="rehobothokoibu@gmail.com",
     description="Dynamic documentation fetching CLI that converts entire documentation sites to single markdown files for AI/LLM consumption",