llm-docs-builder 0.9.4 → 0.11.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 66983a07a7271c966999350d03fbde1b1080ef0ac05a7209452cbb8720074e1b
-  data.tar.gz: 5d7cb81a700db6a43c17145af56ffcd2f6cea4a9a5182d4a2b14fd7772c8ee07
+  metadata.gz: deeae74a329018b4a43d7845a3be8b7c31347699c3ff7abd93d7b697a48982a3
+  data.tar.gz: f9c842caa93a45b4d75c45a15e116e6f98d8463e5268e2de32e498c725877e4f
 SHA512:
-  metadata.gz: 416b6f94c7e7dbac3bf3e6ae8793adcf9893d685aec29cc83665c2f9a6ab312bd422542e0074bbeec961f23740698aac6e517ebb7c87f3cb0fef3c8c6067c662
-  data.tar.gz: 2fba65092d82dbbeea60ce05317a781ed82f20367313d4eab9a69425e6e51f70ec106ca5f56218bbe255d1485752732de2a876359fde7018e706b948d51053fb
+  metadata.gz: 94575eced147bd6740b5395acd41d3f46ffcadf40908831df081d5e03f56b35a2e1e9acfdfc7642af775b2aa86fe48ea322dd11baf48512d2f2ef43a1a491079
+  data.tar.gz: 52b7d40d4a95acd20a408f4d453f32c7154637ce42ec210cf67010ce10dbe14ad711c4cbfd4060d8ce5018b91668315d552732ea7118374e69228a869792ff0f

data/.github/workflows/push.yml

@@ -24,7 +24,7 @@ jobs:
           fetch-depth: 0
 
       - name: Set up Ruby
-        uses: ruby/setup-ruby@ab177d40ee5483edb974554986f56b33477e21d0 # v1.265.0
+        uses: ruby/setup-ruby@4ff6f3611a42bc75eee1e5138240eb1613f48c8f # v1.266.0
         with:
           bundler-cache: false
 

data/AGENTS.md

@@ -0,0 +1,20 @@
+# Repository Guidelines
+
+## Project Structure & Module Organization
+Core gem code lives in `lib/llm_docs_builder`, with single-responsibility modules such as `generator.rb`, `validator.rb`, and the CLI glue in `cli.rb`. Shared entrypoint `lib/llm_docs_builder.rb` wires dependencies. Executables reside in `bin/`: `llm-docs-builder` boots the CLI, while `rspecs` runs the full test matrix. Specs mirror library files under `spec/` with command-level coverage in `spec/integrations`. Static assets (logos, diff screenshots) are in `misc/`. Example configuration templates live at `llm-docs-builder.yml.example`.
+
+## Build, Test, and Development Commands
+- `bundle install` — sync gem dependencies defined in `Gemfile`.
+- `bundle exec rake` — default task; runs RSpec and RuboCop together.
+- `bundle exec rspec` or `bin/rspecs` — execute unit and integration specs with doc formatter.
+- `bundle exec rubocop` — enforce the Ruby style guide; mirrors CI.
+- `bin/llm-docs-builder transform --docs README.md` — smoke-test the CLI against a local file.
+
+## Coding Style & Naming Conventions
+Target Ruby 3.2 with two-space indentation and trailing newline. Prefer single-quoted strings; enable `# frozen_string_literal: true` headers on Ruby files. Keep lines ≤120 characters except where the RuboCop config allows. Use descriptive module/class names (e.g., `LlmDocsBuilder::Generator`) and predicate methods ending with `?` when returning booleans. Place supporting fixtures in `spec/support` if added, and name files after the class they extend.
+
+## Testing Guidelines
+RSpec is the sole testing framework. Name files `*_spec.rb` and align describe blocks with constant paths. Integration scenarios belong in `spec/integrations` to capture CLI behaviors. SimpleCov is enabled by default for line and branch coverage; export `SIMPLECOV=false` for quick local runs. Persist example statuses with the automatically managed `spec/examples.txt`.
+
+## Commit & Pull Request Guidelines
+Keep commit subjects short, present-tense, and focused (e.g., `Align CLI config (#27)`). Group related changes together so `git log` remains readable. Pull requests should describe motivation, summarize behavioral impact, link related issues or discussions, and include CLI output or screenshots when touching generated docs. Ensure CI passes (`bundle exec rake`) before requesting review, and note any follow-up work in the PR description.

data/CHANGELOG.md

@@ -1,5 +1,25 @@
 # Changelog
 
+## 0.11.0 (2025-11-03)
+- [Feature] **Transform from URL** — The `transform` command now accepts a remote URL via `--url` and processes fetched content through the standard transformer pipeline.
+  - Example: `llm-docs-builder transform --url https://example.com/docs/page.html`
+  - Applies all configured transformations and output options identically to local files
+  - By @Eric-Guo and @codex in PR #28.
+
+## 0.10.0 (2025-10-27)
+- [Feature] **llms.txt Specification Compliance** - Updated output format to fully comply with the llms.txt specification from llmstxt.org.
+  - **Metadata Format**: Metadata now appears within the description field using parentheses and comma separators: `- [title](url): description (tokens:450, updated:2025-10-13, priority:high)`
+  - **Optional Descriptions**: Parser now correctly handles links without descriptions: `- [title](url)` per spec
+  - **Multi-Section Support**: Documents automatically organized into `Documentation`, `Examples`, and `Optional` sections based on priority
+  - **Body Content Support**: Added optional `body` config parameter for custom content between description and sections
+  - Priority-based categorization: 1-3 → Documentation, 4-5 → Examples, 6-7 → Optional
+  - Empty sections are automatically omitted from output
+  - Updated parser regex from `/^[-*]\s*\[([^\]]+)\]\(([^)]+)\):\s*(.*)$/m` to `/^[-*]\s*\[([^\]]+)\]\(([^)]+)\)(?::\s*([^\n]*))?$/` to make descriptions optional
+  - Fixed multiline regex greedy matching issue that was capturing only one link per section
+- [Test] Added comprehensive test suite for spec compliance (8 new parser tests, 7 new generator tests)
+- [Docs] Updated README with multi-section organization examples and body content usage
+- **Breaking Change**: Metadata format has changed from `tokens:450 updated:2025-10-13` to `(tokens:450, updated:2025-10-13)` for spec compliance
+
 ## 0.9.4 (2025-10-27)
 - [Feature] **Auto-Exclude Hidden Directories** - Hidden directories (starting with `.`) are now automatically excluded by default to prevent noise from `.git`, `.lint`, `.github`, etc.
   - Adds `include_hidden: false` as default behavior

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    llm-docs-builder (0.9.4)
+    llm-docs-builder (0.11.0)
       zeitwerk (~> 2.6)
 
 GEM
@@ -12,15 +12,15 @@ GEM
     coderay (1.1.3)
     diff-lcs (1.6.2)
     docile (1.4.1)
-    json (2.13.2)
+    json (2.15.2)
     language_server-protocol (3.17.0.5)
     lint_roller (1.1.0)
     method_source (1.1.0)
     parallel (1.27.0)
-    parser (3.3.9.0)
+    parser (3.3.10.0)
       ast (~> 2.4.1)
       racc
-    prism (1.4.0)
+    prism (1.6.0)
     pry (0.15.2)
       coderay (~> 1.1)
       method_source (~> 1.0)
@@ -29,22 +29,22 @@ GEM
       pry (>= 0.13, < 0.16)
     racc (1.8.1)
     rainbow (3.1.1)
-    rake (13.3.0)
-    regexp_parser (2.11.2)
-    rspec (3.13.1)
+    rake (13.3.1)
+    regexp_parser (2.11.3)
+    rspec (3.13.2)
       rspec-core (~> 3.13.0)
       rspec-expectations (~> 3.13.0)
       rspec-mocks (~> 3.13.0)
-    rspec-core (3.13.5)
+    rspec-core (3.13.6)
       rspec-support (~> 3.13.0)
     rspec-expectations (3.13.5)
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.13.0)
-    rspec-mocks (3.13.5)
+    rspec-mocks (3.13.6)
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.13.0)
-    rspec-support (3.13.5)
-    rubocop (1.80.0)
+    rspec-support (3.13.6)
+    rubocop (1.81.6)
       json (~> 2.3)
       language_server-protocol (~> 3.17.0.2)
       lint_roller (~> 1.1.0)
@@ -52,10 +52,10 @@ GEM
       parser (>= 3.3.0.2)
       rainbow (>= 2.2.2, < 4.0)
       regexp_parser (>= 2.9.3, < 3.0)
-      rubocop-ast (>= 1.46.0, < 2.0)
+      rubocop-ast (>= 1.47.1, < 2.0)
       ruby-progressbar (~> 1.7)
       unicode-display_width (>= 2.4.0, < 4.0)
-    rubocop-ast (1.46.0)
+    rubocop-ast (1.47.1)
       parser (>= 3.3.7.2)
       prism (~> 1.4)
     ruby-progressbar (1.13.0)
@@ -65,9 +65,9 @@ GEM
       simplecov_json_formatter (~> 0.1)
     simplecov-html (0.13.2)
     simplecov_json_formatter (0.1.4)
-    unicode-display_width (3.1.5)
-      unicode-emoji (~> 4.0, >= 4.0.4)
-    unicode-emoji (4.0.4)
+    unicode-display_width (3.2.0)
+      unicode-emoji (~> 4.1)
+    unicode-emoji (4.1.0)
     zeitwerk (2.7.3)
 
 PLATFORMS
@@ -85,4 +85,4 @@ DEPENDENCIES
   simplecov (~> 0.21)
 
 BUNDLED WITH
-   2.7.1
+   2.7.2

data/README.md

@@ -61,6 +61,9 @@ Factor:         2.8x smaller
 # Single file
 llm-docs-builder transform --docs README.md
 
+# Fetch and transform a remote page
+llm-docs-builder transform --url https://yoursite.com/docs/page.html
+
 # Bulk transform with config
 llm-docs-builder bulk-transform --config llm-docs-builder.yml
 ```
@@ -109,6 +112,7 @@ docs: ./docs
 base_url: https://myproject.io
 title: My Project
 description: Brief description
+body: Optional body content between description and sections
 output: llms.txt
 suffix: .llm
 verbose: false
@@ -289,9 +293,9 @@ Generate enriched llms.txt files with token counts, timestamps, and priority lab
 
 **Enhanced llms.txt (with metadata enabled):**
 ```markdown
-- [Getting Started](https://myproject.io/docs/Getting-Started.md) tokens:450 updated:2025-10-13 priority:high
-- [Configuration](https://myproject.io/docs/Configuration.md) tokens:2800 updated:2025-10-12 priority:high
-- [Advanced Topics](https://myproject.io/docs/Advanced.md) tokens:5200 updated:2025-09-15 priority:medium
+- [Getting Started](https://myproject.io/docs/Getting-Started.md): Quick start guide (tokens:450, updated:2025-10-13, priority:high)
+- [Configuration](https://myproject.io/docs/Configuration.md): Configuration options (tokens:2800, updated:2025-10-12, priority:high)
+- [Advanced Topics](https://myproject.io/docs/Advanced.md): Deep dive topics (tokens:5200, updated:2025-09-15, priority:medium)
 ```
 
 **Benefits:**
@@ -309,6 +313,68 @@ include_priority: true      # Show priority labels (high/medium/low)
 calculate_compression: true # Show compression ratios (slower, requires transformation)
 ```
 
+**Note:** Metadata is formatted according to the llms.txt specification, appearing within the description field using parentheses and comma separators for spec compliance.
+
+### Multi-Section Organization
+
+Documents are automatically organized into multiple sections based on priority, following the llms.txt specification:
+
+**Priority-based categorization:**
+- **Documentation** (priority 1-3): Essential docs like README, getting started guides, user guides
+- **Examples** (priority 4-5): Tutorials and example files
+- **Optional** (priority 6-7): Advanced topics and reference documentation
+
+**Example output:**
+```markdown
+# My Project
+
+> Project description
+
+## Documentation
+
+- [README](README.md): Main documentation
+- [Getting Started](getting-started.md): Quick start guide
+
+## Examples
+
+- [Basic Tutorial](tutorial.md): Step-by-step tutorial
+- [Code Examples](examples.md): Example code
+
+## Optional
+
+- [Advanced Topics](advanced.md): Deep dive into advanced features
+- [API Reference](reference.md): Complete API reference
+```
+
+Empty sections are automatically omitted. The "Optional" section aligns with the llms.txt spec for marking secondary content that can be skipped when context windows are limited.
+
+### Body Content
+
+Add custom body content between the description and documentation sections:
+
+```yaml
+# llm-docs-builder.yml
+title: My Project
+description: Brief description
+body: |
+  This framework is built on Ruby and focuses on performance.
+  Key concepts: streaming, batching, and parallel processing.
+docs: ./docs
+```
+
+This produces:
+```markdown
+# My Project
+
+> Brief description
+
+This framework is built on Ruby and focuses on performance.
+Key concepts: streaming, batching, and parallel processing.
+
+## Documentation
+...
+```
+
 ## Advanced Compression Options
 
 All compression features can be used individually for fine-grained control:

data/lib/llm_docs_builder/cli.rb

@@ -68,8 +68,9 @@ module LlmDocsBuilder
     # @param argv [Array<String>] command-line arguments
     # @return [Hash] parsed options including :command, :config, :docs, :output, :verbose
     def parse_options(argv)
+      command_token = argv.first
       options = {
-        command: argv.first&.match?(/^[a-z-]+$/) ? argv.shift : nil
+        command: command_token&.match?(/\A[a-z](?:[a-z-]*[a-z])?\z/) ? argv.shift : nil
       }
 
       OptionParser.new do |opts|
@@ -100,7 +101,7 @@ module LlmDocsBuilder
           options[:output] = path
         end
 
-        opts.on('-u', '--url URL', 'URL to fetch for comparison') do |url|
+        opts.on('-u', '--url URL', 'URL to fetch for transform or comparison') do |url|
           options[:url] = url
         end
 
@@ -185,21 +186,42 @@ module LlmDocsBuilder
       config = LlmDocsBuilder::Config.new(options[:config])
       merged_options = config.merge_with_options(options)
 
-      file_path = merged_options[:docs]
+      url = options[:url]
+      cli_file_path = options[:docs]
+      config_file_path = config['docs']
+      file_path = url ? cli_file_path : (cli_file_path || config_file_path)
 
-      unless file_path
-        puts 'File path required for transform command (use -d/--docs)'
+      if url && cli_file_path
+        puts 'Cannot use both --docs and --url for transform command'
         exit 1
       end
 
-      unless File.exist?(file_path)
-        puts "File not found: #{file_path}"
-        exit 1
+      unless file_path
+        unless url
+          puts 'File path required for transform command (use -d/--docs)'
+          exit 1
+        end
       end
 
-      puts "Transforming #{file_path}..." if merged_options[:verbose]
+      content =
+        if url
+          puts "Fetching #{url}..." if merged_options[:verbose]
+          fetcher = LlmDocsBuilder::UrlFetcher.new(verbose: merged_options[:verbose])
+          remote_content = fetcher.fetch(url)
+          puts "Transforming content from #{url}..." if merged_options[:verbose]
+          transform_options = merged_options.merge(content: remote_content, docs: nil, source_url: url)
+          LlmDocsBuilder.transform_markdown(nil, transform_options)
+        else
+          unless File.exist?(file_path)
+            puts "File not found: #{file_path}"
+            exit 1
+          end
 
-      content = LlmDocsBuilder.transform_markdown(file_path, merged_options)
+          puts "Transforming #{file_path}..." if merged_options[:verbose]
+
+          merged_options[:docs] = file_path
+          LlmDocsBuilder.transform_markdown(file_path, merged_options)
+        end
 
       if merged_options[:output] && merged_options[:output] != 'llms.txt'
         File.write(merged_options[:output], content)

data/lib/llm_docs_builder/comparator.rb

@@ -1,8 +1,5 @@
 # frozen_string_literal: true
 
-require 'net/http'
-require 'uri'
-
 module LlmDocsBuilder
   # Compares content sizes between human and AI versions
   #
@@ -30,7 +27,7 @@ module LlmDocsBuilder
     AI_USER_AGENT = 'Claude-Web/1.0 (Anthropic AI Assistant)'
 
     # Maximum number of redirects to follow before raising an error
-    MAX_REDIRECTS = 10
+    MAX_REDIRECTS = UrlFetcher::MAX_REDIRECTS
 
     # @return [String] URL to compare
     attr_reader :url
@@ -133,78 +130,11 @@ module LlmDocsBuilder
     # @return [String] response body
     # @raise [Errors::GenerationError] if fetch fails or too many redirects
     def fetch_url(url_string, user_agent, redirect_count = 0)
-      if redirect_count >= MAX_REDIRECTS
-        raise(
-          Errors::GenerationError,
-          "Too many redirects (#{MAX_REDIRECTS}) when fetching #{url_string}"
-        )
-      end
-
-      uri = validate_and_parse_url(url_string)
-
-      http = Net::HTTP.new(uri.host, uri.port)
-      http.use_ssl = uri.scheme == 'https'
-      http.open_timeout = 10
-      http.read_timeout = 30
-
-      request = Net::HTTP::Get.new(uri.request_uri)
-      request['User-Agent'] = user_agent
-
-      response = http.request(request)
-
-      case response
-      when Net::HTTPSuccess
-        response.body
-      when Net::HTTPRedirection
-        # Follow redirect with incremented counter
-        redirect_url = response['location']
-        puts "  Redirecting to #{redirect_url}..." if options[:verbose] && redirect_count.positive?
-        fetch_url(redirect_url, user_agent, redirect_count + 1)
-      else
-        raise(
-          Errors::GenerationError,
-          "Failed to fetch #{url_string}: #{response.code} #{response.message}"
-        )
-      end
-    rescue Errors::GenerationError
-      raise
-    rescue StandardError => e
-      raise(
-        Errors::GenerationError,
-        "Error fetching #{url_string}: #{e.message}"
-      )
-    end
-
-    # Validates and parses URL to prevent malformed URLs
-    #
-    # @param url_string [String] URL to validate and parse
-    # @return [URI::HTTP, URI::HTTPS] parsed URI
-    # @raise [Errors::GenerationError] if URL is invalid or uses unsupported scheme
-    def validate_and_parse_url(url_string)
-      uri = URI.parse(url_string)
-
-      # Only allow HTTP and HTTPS schemes
-      unless %w[http https].include?(uri.scheme&.downcase)
-        raise(
-          Errors::GenerationError,
-          "Unsupported URL scheme: #{uri.scheme || 'none'} (only http/https allowed)"
-        )
-      end
-
-      # Ensure host is present
-      if uri.host.nil? || uri.host.empty?
-        raise(
-          Errors::GenerationError,
-          "Invalid URL: missing host in #{url_string}"
-        )
-      end
-
-      uri
-    rescue URI::InvalidURIError => e
-      raise(
-        Errors::GenerationError,
-        "Invalid URL format: #{e.message}"
+      fetcher = UrlFetcher.new(
+        user_agent: user_agent,
+        verbose: options[:verbose]
       )
+      fetcher.fetch(url_string, redirect_count)
     end
 
     # Calculate comparison statistics

data/lib/llm_docs_builder/config.rb

@@ -57,10 +57,15 @@ module LlmDocsBuilder
     def merge_with_options(options)
       # CLI options override config file, config file provides defaults
       {
-        docs: options[:docs] || self['docs'] || '.',
+        docs: if options.key?(:docs)
+                options[:docs]
+              else
+                self['docs'] || '.'
+              end,
         base_url: options[:base_url] || self['base_url'],
         title: options[:title] || self['title'],
         description: options[:description] || self['description'],
+        body: options[:body] || self['body'],
         output: options[:output] || self['output'] || 'llms.txt',
         convert_urls: if options.key?(:convert_urls)
                         options[:convert_urls]
@@ -170,7 +175,10 @@ module LlmDocsBuilder
                                else
                                  self['calculate_compression'] || false
                                end
-      }
+      }.tap do |merged|
+        merged[:content] = options[:content] if options.key?(:content)
+        merged[:source_url] = options[:source_url] if options.key?(:source_url)
+      end
     end
 
     # Check if a config file was found and exists

data/lib/llm_docs_builder/generator.rb

@@ -210,7 +210,11 @@ module LlmDocsBuilder
 
     # Constructs llms.txt content from analyzed documentation files
     #
-    # Combines title, description, and documentation links into formatted output
+    # Combines title, description, body content, and documentation links into formatted output.
+    # Organizes documents into sections based on priority:
+    # - Priority 1-3: Documentation (essential docs like README, getting started)
+    # - Priority 4-5: Examples (tutorials, example files)
+    # - Priority 6-7: Optional (advanced topics, reference docs)
     #
     # @param docs [Array<Hash>] analyzed file metadata
     # @return [String] formatted llms.txt content
@@ -224,31 +228,60 @@ module LlmDocsBuilder
       content << "> #{description}" if description
       content << ''
 
-      if docs.any?
-        content << '## Documentation'
+      # Add optional body content
+      if options[:body] && !options[:body].empty?
+        content << options[:body]
         content << ''
+      end
 
-        docs.each do |doc|
-          url = build_url(doc[:path])
-          line = if doc[:description] && !doc[:description].empty?
-                   "- [#{doc[:title]}](#{url}): #{doc[:description]}"
-                 else
-                   "- [#{doc[:title]}](#{url})"
-                 end
-
-          # Append metadata if enabled
-          if options[:include_metadata]
-            metadata_parts = []
-            metadata_parts << "tokens:#{doc[:tokens]}" if doc[:tokens]
-            metadata_parts << "compression:#{doc[:compression]}" if doc[:compression]
-            metadata_parts << "updated:#{doc[:updated]}" if doc[:updated]
-            metadata_parts << priority_label(doc[:priority]) if options[:include_priority]
-
-            line += " #{metadata_parts.join(' ')}" unless metadata_parts.empty?
+      if docs.any?
+        # Categorize docs by priority into sections
+        sections = {
+          'Documentation' => docs.select { |d| d[:priority] <= 3 },
+          'Examples' => docs.select { |d| d[:priority] >= 4 && d[:priority] <= 5 },
+          'Optional' => docs.select { |d| d[:priority] >= 6 }
+        }
+
+        # Build each section (skip empty ones)
+        sections.each do |section_name, section_docs|
+          next if section_docs.empty?
+
+          content << "## #{section_name}"
+          content << ''
+
+          section_docs.each do |doc|
+            url = build_url(doc[:path])
+
+            # Build metadata string if enabled
+            metadata_str = nil
+            if options[:include_metadata]
+              metadata_parts = []
+              metadata_parts << "tokens:#{doc[:tokens]}" if doc[:tokens]
+              metadata_parts << "compression:#{doc[:compression]}" if doc[:compression]
+              metadata_parts << "updated:#{doc[:updated]}" if doc[:updated]
+              metadata_parts << priority_label(doc[:priority]) if options[:include_priority]
+
+              metadata_str = "(#{metadata_parts.join(', ')})" unless metadata_parts.empty?
+            end
+
+            # Build line according to spec: - [title](url): description (metadata)
+            line = if doc[:description] && !doc[:description].empty?
+                     base = "- [#{doc[:title]}](#{url}): #{doc[:description]}"
+                     metadata_str ? "#{base} #{metadata_str}" : base
+                   else
+                     # No description: - [title](url) (metadata)
+                     base = "- [#{doc[:title]}](#{url})"
+                     metadata_str ? "#{base}: #{metadata_str}" : base
+                   end
+
+            content << line
           end
 
-          content << line
+          content << ''
         end
+
+        # Remove trailing empty line
+        content.pop if content.last == ''
       end
 
       "#{content.join("\n")}\n"

data/lib/llm_docs_builder/markdown_transformer.rb

@@ -55,7 +55,7 @@ module LlmDocsBuilder
     #
     # @return [String] transformed markdown content
     def transform
-      content = File.read(file_path)
+      content = load_content
 
       # Build and execute transformation pipeline
       content = cleanup_transformer.transform(content, options)
@@ -124,5 +124,16 @@ module LlmDocsBuilder
       }
       compressor.compress(content, compression_methods)
     end
+
+    # Load source content either from provided string or file path
+    #
+    # @return [String] markdown content to transform
+    def load_content
+      if options[:content]
+        options[:content].dup
+      else
+        File.read(file_path)
+      end
+    end
   end
 end

data/lib/llm_docs_builder/parser.rb

@@ -85,7 +85,7 @@ module LlmDocsBuilder
 
     # Extracts markdown links from section content into structured format
     #
-    # Scans for markdown list items with links and descriptions. Returns raw content
+    # Scans for markdown list items with links and optional descriptions. Returns raw content
     # if no links are found in the expected format.
     #
     # @param content [String] raw section content
@@ -93,11 +93,13 @@ module LlmDocsBuilder
     def parse_section_content(content)
       links = []
 
-      content.scan(/^[-*]\s*\[([^\]]+)\]\(([^)]+)\):\s*(.*)$/m) do |title, url, description|
+      # Updated regex: description is optional (non-capturing group with ?)
+      # Use [^\n]* instead of .* to avoid matching across lines
+      content.scan(/^[-*]\s*\[([^\]]+)\]\(([^)]+)\)(?::\s*([^\n]*))?$/) do |title, url, description|
         links << {
           title: title,
           url: url,
-          description: description.strip
+          description: description&.strip || ''
         }
       end
 

data/lib/llm_docs_builder/url_fetcher.rb

@@ -0,0 +1,120 @@
+# frozen_string_literal: true
+
+require 'net/http'
+require 'uri'
+
+module LlmDocsBuilder
+  # Lightweight HTTP client for fetching remote documentation pages.
+  #
+  # Provides common functionality needed by multiple commands (transform, compare)
+  # including strict scheme validation, redirect handling and sensible timeouts.
+  class UrlFetcher
+    DEFAULT_USER_AGENT = 'llm-docs-builder/1.0 (+https://github.com/mensfeld/llm-docs-builder)'
+    MAX_REDIRECTS = 10
+
+    # @param user_agent [String] HTTP user agent header value
+    # @param verbose [Boolean] enable redirect logging
+    # @param output [IO] IO stream used for redirect logging
+    def initialize(user_agent: DEFAULT_USER_AGENT, verbose: false, output: $stdout)
+      @user_agent = user_agent
+      @verbose = verbose
+      @output = output
+    end
+
+    # Fetch remote URL content while following redirects.
+    #
+    # @param url_string [String] URL to fetch
+    # @param redirect_count [Integer] current redirect depth (internal use)
+    # @return [String] response body
+    # @raise [Errors::GenerationError] on invalid URLs, network failures, or redirect loops
+    def fetch(url_string, redirect_count = 0)
+      if redirect_count >= MAX_REDIRECTS
+        raise(
+          Errors::GenerationError,
+          "Too many redirects (#{MAX_REDIRECTS}) when fetching #{url_string}"
+        )
+      end
+
+      uri = validate_and_parse_url(url_string)
+
+      http = Net::HTTP.new(uri.host, uri.port)
+      http.use_ssl = uri.scheme == 'https'
+      http.open_timeout = 10
+      http.read_timeout = 30
+
+      request = Net::HTTP::Get.new(uri.request_uri)
+      request['User-Agent'] = @user_agent
+
+      response = http.request(request)
+
+      case response
+      when Net::HTTPSuccess
+        response.body
+      when Net::HTTPRedirection
+        redirect_url = absolute_redirect_url(uri, response['location'])
+        log_redirect(redirect_url)
+        fetch(redirect_url, redirect_count + 1)
+      else
+        raise(
+          Errors::GenerationError,
+          "Failed to fetch #{url_string}: #{response.code} #{response.message}"
+        )
+      end
+    rescue Errors::GenerationError
+      raise
+    rescue StandardError => e
+      raise(
+        Errors::GenerationError,
+        "Error fetching #{url_string}: #{e.message}"
+      )
+    end
+
+    private
+
+    def validate_and_parse_url(url_string)
+      uri = URI.parse(url_string)
+
+      unless %w[http https].include?(uri.scheme&.downcase)
+        raise(
+          Errors::GenerationError,
+          "Unsupported URL scheme: #{uri.scheme || 'none'} (only http/https allowed)"
+        )
+      end
+
+      if uri.host.nil? || uri.host.empty?
+        raise(
+          Errors::GenerationError,
+          "Invalid URL: missing host in #{url_string}"
+        )
+      end
+
+      uri
+    rescue URI::InvalidURIError => e
+      raise(
+        Errors::GenerationError,
+        "Invalid URL format: #{e.message}"
+      )
+    end
+
+    def absolute_redirect_url(base_uri, location)
+      raise(
+        Errors::GenerationError,
+        "Redirect missing location header for #{base_uri}"
+      ) if location.nil? || location.empty?
+
+      URI.join(base_uri, location).to_s
+    rescue URI::InvalidURIError => e
+      raise(
+        Errors::GenerationError,
+        "Invalid redirect URL from #{base_uri}: #{e.message}"
+      )
+    end
+
+    def log_redirect(url)
+      return unless @verbose
+
+      @output.puts("  Redirecting to #{url}...")
+    end
+  end
+end
+

data/lib/llm_docs_builder/version.rb

@@ -2,5 +2,5 @@
 
 module LlmDocsBuilder
   # Current version of the LlmDocsBuilder gem
-  VERSION = '0.9.4'
+  VERSION = '0.11.0'
 end

data/lib/llm_docs_builder.rb

@@ -25,6 +25,7 @@ module LlmDocsBuilder
     # @option options [Boolean] :convert_urls convert HTML URLs to markdown format (overrides
     #   config)
     # @option options [Boolean] :verbose enable verbose output (overrides config)
+    # @option options [String] :content raw markdown content (used for remote sources)
     # @return [String] generated llms.txt content
     #
     # @example Generate from docs directory

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: llm-docs-builder
 version: !ruby/object:Gem::Version
-  version: 0.9.4
+  version: 0.11.0
 platform: ruby
 authors:
 - Maciej Mensfeld
@@ -116,6 +116,7 @@ files:
 - ".rspec"
 - ".rubocop.yml"
 - ".ruby-version"
+- AGENTS.md
 - CHANGELOG.md
 - Dockerfile
 - Gemfile
@@ -143,6 +144,7 @@ files:
 - lib/llm_docs_builder/transformers/heading_transformer.rb
 - lib/llm_docs_builder/transformers/link_transformer.rb
 - lib/llm_docs_builder/transformers/whitespace_transformer.rb
+- lib/llm_docs_builder/url_fetcher.rb
 - lib/llm_docs_builder/validator.rb
 - lib/llm_docs_builder/version.rb
 - llm-docs-builder.gemspec