doc-fetch-cli 1.1.0 → 1.1.1

package/CONTRIBUTING.md

@@ -0,0 +1,274 @@
+# Contributing to DocFetch
+
+Thank you for your interest in contributing to DocFetch! 🎉 This guide will help you get started.
+
+## 📋 Quick Overview
+
+1. **Create an issue first** - Always start by opening an issue to discuss your change
+2. **Wait for feedback** - Maintainers will respond and provide guidance
+3. **Fork and develop** - Once approved, fork the repo and make your changes
+4. **Submit a PR** - Open a pull request referencing the issue
+5. **Review process** - Maintainers will review and provide feedback
+6. **Merge** - Once approved, your contribution will be merged!
+
+---
+
+## 🐛 Before You Start: Create an Issue
+
+**⚠️ IMPORTANT: Always create an issue before submitting a PR!**
+
+This helps us:
+- Avoid duplicate work
+- Discuss the best approach
+- Ensure your contribution aligns with project goals
+- Get early feedback from maintainers
+
+### Types of Issues
+
+#### 🐞 Bug Reports
+Include:
+- Clear description of the bug
+- Steps to reproduce
+- Expected vs actual behavior
+- Environment details (OS, Go version, DocFetch version)
+- Sample command that triggers the bug
+- Error messages or logs
+
+#### ✨ Feature Requests
+Include:
+- Clear description of the feature
+- Use case / problem it solves
+- Example usage
+- Any relevant links or references
+
+#### 📝 Documentation Improvements
+Include:
+- What needs improvement
+- Why it's needed
+- Suggested changes
+
+#### ⚡ Performance Improvements
+Include:
+- Current performance metrics
+- Proposed improvements
+- Benchmark results (if available)
+
+---
+
+## 🚀 Development Setup
+
+### Prerequisites
+
+- Go 1.21 or later
+- Git
+- Make (optional, for running tests)
+
+### Fork and Clone
+
+```bash
+# Fork the repository on GitHub, then:
+git clone https://github.com/YOUR_USERNAME/doc-fetch.git
+cd doc-fetch
+
+# Add upstream remote
+git remote add upstream https://github.com/AlphaTechini/doc-fetch.git
+```
+
+### Build from Source
+
+```bash
+# Build the binary
+go build -o doc-fetch ./cmd/docfetch
+
+# Test it works
+./doc-fetch --help
+```
+
+### Run Tests
+
+```bash
+# Run all tests
+go test ./...
+
+# Run tests with coverage
+go test -cover ./...
+
+# Run specific package tests
+go test ./pkg/fetcher/...
+```
+
+---
+
+## 💻 Making Changes
+
+### Branch Naming
+
+Use descriptive branch names:
+- `fix/content-extraction-bug`
+- `feat/add-pdf-support`
+- `docs/update-readme-examples`
+- `perf/improve-concurrent-fetching`
+
+### Code Style
+
+Follow Go best practices:
+- Run `go fmt` before committing
+- Run `go vet` to catch issues
+- Write clear, concise comments
+- Keep functions small and focused
+- Use meaningful variable names
+
+### Testing Requirements
+
+- Add tests for new features
+- Ensure existing tests pass
+- Include edge cases
+- Test with real documentation sites
+
+Example test:
+```go
+func TestContentExtraction(t *testing.T) {
+    doc := createTestDocument()
+    content := cleanContent(doc)
+    
+    if len(content) == 0 {
+        t.Error("Expected content to be extracted")
+    }
+    
+    if !strings.Contains(content, "expected text") {
+        t.Error("Expected content to contain specific text")
+    }
+}
+```
+
+---
+
+## 📤 Submitting a Pull Request
+
+### PR Checklist
+
+Before submitting your PR, ensure:
+
+- [ ] You created an issue first and referenced it in the PR
+- [ ] Your code follows Go style guidelines
+- [ ] All tests pass (`go test ./...`)
+- [ ] You've added tests for new functionality
+- [ ] You've updated documentation if needed
+- [ ] Your commit messages are clear and descriptive
+- [ ] You've rebased on the latest main branch
+
+### PR Template
+
+When creating your PR, include:
+
+```markdown
+## Description
+Brief description of changes
+
+## Related Issue
+Fixes #123 (or "Related to #123")
+
+## Type of Change
+- [ ] Bug fix
+- [ ] New feature
+- [ ] Breaking change
+- [ ] Documentation update
+- [ ] Performance improvement
+- [ ] Refactoring
+
+## Testing
+Describe how you tested this:
+- [ ] Unit tests added/updated
+- [ ] Manual testing with real docs
+- [ ] Tested on: [list platforms]
+
+## Example Usage
+Show example command and output if applicable
+
+## Checklist
+- [ ] Code follows project guidelines
+- [ ] Self-review completed
+- [ ] Comments added where needed
+- [ ] Tests pass locally
+```
+
+---
+
+## 🔍 Review Process
+
+1. **Automated Checks**: CI runs tests and linting
+2. **Maintainer Review**: At least one maintainer reviews
+3. **Feedback**: You may be asked to make changes
+4. **Approval**: Once approved, PR is merged
+5. **Release**: Changes included in next release
+
+Typical timeline: 3-7 days for review
+
+---
+
+## 📖 Contribution Ideas
+
+Looking for ways to contribute? Here are some ideas:
+
+### Easy Wins
+- Fix typos in documentation
+- Add more examples to README
+- Improve error messages
+- Add unit tests for existing code
+
+### Intermediate
+- Add support for new documentation site formats
+- Improve content extraction selectors
+- Add progress indicators
+- Enhance LLM.txt generation
+
+### Advanced
+- Add PDF export support
+- Implement incremental updates
+- Add authentication support for private docs
+- Create plugin system for custom extractors
+
+---
+
+## 🤝 Community Guidelines
+
+### Be Respectful
+- Treat everyone with respect
+- Welcome newcomers
+- Provide constructive feedback
+- Assume good intentions
+
+### Communication
+- Use clear, concise language
+- Explain your reasoning
+- Ask questions if unsure
+- Respond to feedback promptly
+
+### Collaboration
+- Work with maintainers, not against them
+- Be open to suggestions
+- Help other contributors
+- Share knowledge
+
+---
+
+## 📜 License
+
+By contributing to DocFetch, you agree that your contributions will be licensed under the MIT License.
+
+---
+
+## ❓ Questions?
+
+- **General questions**: Open a discussion on GitHub
+- **Bug reports**: Create an issue
+- **Feature requests**: Create an issue
+- **Quick questions**: Check existing issues/discussions first
+
+---
+
+## 🙏 Thank You!
+
+Your contributions make DocFetch better for everyone. Whether it's a typo fix, a new feature, or better documentation - we appreciate your time and effort!
+
+Happy coding! 🚀

package/bin/postinstall.js

@@ -0,0 +1,83 @@
+#!/usr/bin/env node
+/**
+ * Post-install script for doc-fetch-cli
+ * Checks if global bin directory is in PATH and provides helpful instructions
+ */
+
+const { execSync } = require('child_process');
+const path = require('path');
+const os = require('os');
+const fs = require('fs');
+
+console.log('🎉 DocFetch CLI installed successfully!\n');
+
+// Get npm global prefix
+let globalPrefix;
+try {
+    globalPrefix = execSync('npm config get prefix', { encoding: 'utf8' }).trim();
+} catch (error) {
+    console.error('⚠️  Could not determine npm global prefix');
+    globalPrefix = null;
+}
+
+if (globalPrefix) {
+    const binDir = path.join(globalPrefix, 'bin');
+    const isWindows = os.platform() === 'win32';
+    
+    console.log(`📦 Installed to: ${binDir}\n`);
+    
+    // Check if bin directory is in PATH
+    const pathEnv = process.env.PATH || '';
+    const pathDirs = pathEnv.split(isWindows ? ';' : ':');
+    const isInPath = pathDirs.some(dir => path.resolve(dir) === path.resolve(binDir));
+    
+    if (!isInPath) {
+        console.log('⚠️  WARNING: Global bin directory is not in your PATH!\n');
+        console.log('To use doc-fetch-cli, add this directory to your PATH:\n');
+        console.log(`   ${binDir}\n`);
+        
+        // Provide platform-specific instructions
+        const shell = process.env.SHELL || '/bin/bash';
+        const isZsh = shell.includes('zsh');
+        const isBash = shell.includes('bash');
+        
+        console.log('Quick fix:\n');
+        
+        if (isWindows) {
+            console.log('1. Open System Properties → Environment Variables');
+            console.log('2. Edit PATH variable');
+            console.log('3. Add this path:');
+            console.log(`   ${binDir}`);
+            console.log('4. Restart your terminal\n');
+        } else if (isZsh) {
+            console.log('Add this to your ~/.zshrc:');
+            console.log(`   export PATH="${binDir}:$PATH"\n`);
+            console.log('Then run: source ~/.zshrc\n');
+        } else if (isBash) {
+            console.log('Add this to your ~/.bashrc or ~/.bash_profile:');
+            console.log(`   export PATH="${binDir}:$PATH"\n`);
+            console.log('Then run: source ~/.bashrc\n');
+        }
+        
+        console.log('Alternative: Use npx without installing globally\n');
+        console.log('   npx doc-fetch-cli --url https://docs.example.com --output docs.md\n');
+    } else {
+        console.log('✅ Global bin directory is in your PATH\n');
+        console.log('You can now use doc-fetch-cli!\n');
+        console.log('Example usage:');
+        console.log('   doc-fetch --url https://docs.python.org/3 --output docs.md --llm-txt\n');
+        
+        // Test if the command works
+        try {
+            execSync('doc-fetch --version', { encoding: 'utf8', stdio: 'pipe' });
+            console.log('✅ Command verified working!\n');
+        } catch (error) {
+            console.log('⚠️  Command not found in current shell session.\n');
+            console.log('Try running: hash -r  (to clear command cache)\n');
+            console.log('Or restart your terminal.\n');
+        }
+    }
+}
+
+console.log('📚 Documentation: https://github.com/AlphaTechini/doc-fetch\n');
+console.log('✨ Pro tip: Use --llm-txt flag to generate AI-friendly index files!\n');

package/doc_fetch.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: doc-fetch
-Version: 1.1.0
+Version: 1.1.1
 Summary: Dynamic documentation fetching CLI that converts entire documentation sites to single markdown files for AI/LLM consumption
 Home-page: https://github.com/AlphaTechini/doc-fetch
 Author: AlphaTechini

package/doc_fetch.egg-info/SOURCES.txt

@@ -1,7 +1,18 @@
+CONTRIBUTING.md
 README.md
+SECURITY.md
+doc-fetch
+doc-fetch_darwin_amd64
+doc-fetch_linux_amd64
+doc-fetch_windows_amd64.exe
 go.mod
+go.sum
+package.json
 pyproject.toml
 setup.py
+bin/doc-fetch.js
+bin/install.js
+bin/postinstall.js
 cmd/docfetch/main.go
 doc_fetch/__init__.py
 doc_fetch/__main__.py
@@ -14,6 +25,12 @@ doc_fetch.egg-info/not-zip-safe
 doc_fetch.egg-info/top_level.txt
 docs/usage.md
 examples/golang-example.sh
+pkg/fetcher/classifier.go
+pkg/fetcher/describer.go
+pkg/fetcher/extract_nav.go
 pkg/fetcher/fetcher.go
+pkg/fetcher/fetcher_optimized.go
 pkg/fetcher/html2md.go
+pkg/fetcher/llmtxt.go
+pkg/fetcher/validator.go
 pkg/fetcher/writer.go

package/package.json

@@ -1,12 +1,12 @@
 {
   "name": "doc-fetch-cli",
-  "version": "1.1.0",
+  "version": "1.1.1",
   "description": "Dynamic documentation fetching CLI that converts entire documentation sites to single markdown files for AI/LLM consumption",
   "bin": {
     "doc-fetch": "./bin/doc-fetch.js"
   },
   "scripts": {
-    "postinstall": "node ./bin/install.js"
+    "postinstall": "node ./bin/postinstall.js"
   },
   "repository": {
     "type": "git",

package/pkg/fetcher/extract_nav.go

@@ -0,0 +1,163 @@
+package fetcher
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/PuerkitoBio/goquery"
+)
+
+// ExtractNavigationStructure extracts nav elements with h2/h3, ul/li, and hrefs
+func ExtractNavigationStructure(doc *goquery.Document) string {
+	var result strings.Builder
+	
+	result.WriteString("# Navigation Structure\n\n")
+	
+	// Find all nav elements
+	doc.Find("nav").Each(func(i int, nav *goquery.Selection) {
+		result.WriteString(fmt.Sprintf("## Navigation Block %d\n\n", i+1))
+		
+		// Look for headings in nav
+		nav.Find("h1, h2, h3, h4, h5, h6").Each(func(j int, h *goquery.Selection) {
+			tagName := h.Get(0).Data
+			text := strings.TrimSpace(h.Text())
+			if text != "" {
+				result.WriteString(fmt.Sprintf("### %s: %s\n\n", tagName, text))
+			}
+			
+			// Find ul under this heading
+			h.NextFiltered("ul").Each(func(k int, ul *goquery.Selection) {
+				result.WriteString(extractListWithLinks(ul, 1))
+			})
+		})
+		
+		// Also find ul directly in nav
+		nav.ChildrenFiltered("ul").Each(func(k int, ul *goquery.Selection) {
+			result.WriteString(extractListWithLinks(ul, 1))
+		})
+		
+		result.WriteString("---\n\n")
+	})
+	
+	// Also look for elements with navigation-related classes/ids
+	navSelectors := []string{
+		"[class*='nav']",
+		"[id*='nav']",
+		"[class*='menu']",
+		"[id*='menu']",
+		"[role='navigation']",
+		".toc",
+		"#toc",
+		"[class*='toc']",
+		"[id*='toc']",
+	}
+	
+	for _, selector := range navSelectors {
+		doc.Find(selector).Each(func(i int, s *goquery.Selection) {
+			// Skip if already processed as nav element
+			if s.Parent().Is("nav") {
+				return
+			}
+			
+			result.WriteString(fmt.Sprintf("## Navigation Element (matched: %s)\n\n", selector))
+			
+			// Extract headings
+			s.Find("h1, h2, h3, h4, h5, h6").Each(func(j int, h *goquery.Selection) {
+				tagName := h.Get(0).Data
+				text := strings.TrimSpace(h.Text())
+				if text != "" {
+					result.WriteString(fmt.Sprintf("### %s: %s\n\n", tagName, text))
+				}
+			})
+			
+			// Extract lists with links
+			s.Find("ul, ol").Each(func(k int, list *goquery.Selection) {
+				result.WriteString(extractListWithLinks(list, 1))
+			})
+			
+			result.WriteString("---\n\n")
+		})
+	}
+	
+	return result.String()
+}
+
+// extractListWithLinks extracts list items with their href attributes
+func extractListWithLinks(list *goquery.Selection, indentLevel int) string {
+	var result strings.Builder
+	
+	indent := strings.Repeat("  ", indentLevel)
+	
+	list.Find("> li").Each(func(i int, li *goquery.Selection) {
+		// Get the text
+		text := strings.TrimSpace(li.Text())
+		
+		// Find any links in this li
+		li.Find("a[href]").Each(func(j int, a *goquery.Selection) {
+			href, exists := a.Attr("href")
+			linkText := strings.TrimSpace(a.Text())
+			if exists && href != "" {
+				result.WriteString(fmt.Sprintf("%s- [%s](%s)\n", indent, linkText, href))
+			}
+		})
+		
+		// If no links found, just add the text
+		if li.Find("a[href]").Length() == 0 && text != "" {
+			result.WriteString(fmt.Sprintf("%s- %s\n", indent, text))
+		}
+		
+		// Recursively process nested lists
+		li.ChildrenFiltered("ul, ol").Each(func(k int, nested *goquery.Selection) {
+			result.WriteString(extractListWithLinks(nested, indentLevel+1))
+		})
+	})
+	
+	return result.String()
+}
+
+// ExtractAllLinks extracts all links from the page with context
+func ExtractAllLinks(doc *goquery.Document, baseURL string) string {
+	var result strings.Builder
+	
+	result.WriteString("# All Links Found\n\n")
+	
+	linksFound := 0
+	
+	// Group links by section
+	doc.Find("section, article, div[class*='content'], div[id*='content']").Each(func(i int, section *goquery.Selection) {
+		sectionLinks := 0
+		var sectionResult strings.Builder
+		
+		// Get section title
+		title := ""
+		section.Find("h1, h2, h3").First().Each(func(j int, h *goquery.Selection) {
+			title = strings.TrimSpace(h.Text())
+		})
+		
+		if title == "" {
+			title = fmt.Sprintf("Section %d", i+1)
+		}
+		
+		sectionResult.WriteString(fmt.Sprintf("## %s\n\n", title))
+		
+		// Find all links in this section
+		section.Find("a[href]").Each(func(j int, a *goquery.Selection) {
+			href, exists := a.Attr("href")
+			text := strings.TrimSpace(a.Text())
+			if exists && href != "" && text != "" {
+				sectionResult.WriteString(fmt.Sprintf("- [%s](%s)\n", text, href))
+				sectionLinks++
+				linksFound++
+			}
+		})
+		
+		if sectionLinks > 0 {
+			result.WriteString(sectionResult.String())
+			result.WriteString("\n")
+		}
+	})
+	
+	result.WriteString(fmt.Sprintf("\n**Total links found: %d**\n", linksFound))
+	
+	return result.String()
+}

package/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "doc-fetch"
-version = "1.1.0"
+version = "1.1.1"
 description = "Dynamic documentation fetching CLI that converts entire documentation sites to single markdown files for AI/LLM consumption"
 readme = "README.md"
 authors = [{name = "AlphaTechini", email = "rehobothokoibu@gmail.com"}]

package/setup.py

@@ -118,7 +118,7 @@ with open("README.md", "r", encoding="utf-8") as fh:
 
 setup(
     name="doc-fetch",
-    version="1.1.0",
+    version="1.1.1",
     author="AlphaTechini",
     author_email="rehobothokoibu@gmail.com",
     description="Dynamic documentation fetching CLI that converts entire documentation sites to single markdown files for AI/LLM consumption",