llm-docs-builder 0.10.0 → 0.11.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e6407836216f436b728247009f614ea4ea5c2b4de0edf855717129460df4b309
-  data.tar.gz: 8a556fc0b6307529f5c615c05b082521bd021e9e9f34ca30c8bb21d22b21deb2
+  metadata.gz: deeae74a329018b4a43d7845a3be8b7c31347699c3ff7abd93d7b697a48982a3
+  data.tar.gz: f9c842caa93a45b4d75c45a15e116e6f98d8463e5268e2de32e498c725877e4f
 SHA512:
-  metadata.gz: 3a0b657545415c35187fa1595f3fcc5bd5c27a0c1a292bf00635d3c6f14221ac0a254f008f83acc1f7351ef76c6649e6b0540ac585cd64a670b05ef725b0308e
-  data.tar.gz: 68e95142e374ebae3c292db724a9163c396fd423eaa04983bbdaefd6f6cbb0bc72822dfbc5af9fd8f357f1545320db70bf5a6c8a2413fc5f796e73195adcdd13
+  metadata.gz: 94575eced147bd6740b5395acd41d3f46ffcadf40908831df081d5e03f56b35a2e1e9acfdfc7642af775b2aa86fe48ea322dd11baf48512d2f2ef43a1a491079
+  data.tar.gz: 52b7d40d4a95acd20a408f4d453f32c7154637ce42ec210cf67010ce10dbe14ad711c4cbfd4060d8ce5018b91668315d552732ea7118374e69228a869792ff0f

data/.github/workflows/push.yml

@@ -24,7 +24,7 @@ jobs:
           fetch-depth: 0
 
       - name: Set up Ruby
-        uses: ruby/setup-ruby@ab177d40ee5483edb974554986f56b33477e21d0 # v1.265.0
+        uses: ruby/setup-ruby@4ff6f3611a42bc75eee1e5138240eb1613f48c8f # v1.266.0
         with:
           bundler-cache: false
 

data/AGENTS.md

@@ -0,0 +1,20 @@
+# Repository Guidelines
+
+## Project Structure & Module Organization
+Core gem code lives in `lib/llm_docs_builder`, with single-responsibility modules such as `generator.rb`, `validator.rb`, and the CLI glue in `cli.rb`. Shared entrypoint `lib/llm_docs_builder.rb` wires dependencies. Executables reside in `bin/`: `llm-docs-builder` boots the CLI, while `rspecs` runs the full test matrix. Specs mirror library files under `spec/` with command-level coverage in `spec/integrations`. Static assets (logos, diff screenshots) are in `misc/`. Example configuration templates live at `llm-docs-builder.yml.example`.
+
+## Build, Test, and Development Commands
+- `bundle install` — sync gem dependencies defined in `Gemfile`.
+- `bundle exec rake` — default task; runs RSpec and RuboCop together.
+- `bundle exec rspec` or `bin/rspecs` — execute unit and integration specs with doc formatter.
+- `bundle exec rubocop` — enforce the Ruby style guide; mirrors CI.
+- `bin/llm-docs-builder transform --docs README.md` — smoke-test the CLI against a local file.
+
+## Coding Style & Naming Conventions
+Target Ruby 3.2 with two-space indentation and trailing newline. Prefer single-quoted strings; enable `# frozen_string_literal: true` headers on Ruby files. Keep lines ≤120 characters except where the RuboCop config allows. Use descriptive module/class names (e.g., `LlmDocsBuilder::Generator`) and predicate methods ending with `?` when returning booleans. Place supporting fixtures in `spec/support` if added, and name files after the class they extend.
+
+## Testing Guidelines
+RSpec is the sole testing framework. Name files `*_spec.rb` and align describe blocks with constant paths. Integration scenarios belong in `spec/integrations` to capture CLI behaviors. SimpleCov is enabled by default for line and branch coverage; export `SIMPLECOV=false` for quick local runs. Persist example statuses with the automatically managed `spec/examples.txt`.
+
+## Commit & Pull Request Guidelines
+Keep commit subjects short, present-tense, and focused (e.g., `Align CLI config (#27)`). Group related changes together so `git log` remains readable. Pull requests should describe motivation, summarize behavioral impact, link related issues or discussions, and include CLI output or screenshots when touching generated docs. Ensure CI passes (`bundle exec rake`) before requesting review, and note any follow-up work in the PR description.

data/CHANGELOG.md

@@ -1,5 +1,11 @@
 # Changelog
 
+## 0.11.0 (2025-11-03)
+- [Feature] **Transform from URL** — The `transform` command now accepts a remote URL via `--url` and processes fetched content through the standard transformer pipeline.
+  - Example: `llm-docs-builder transform --url https://example.com/docs/page.html`
+  - Applies all configured transformations and output options identically to local files
+  - By @Eric-Guo and @codex in PR #28.
+
 ## 0.10.0 (2025-10-27)
 - [Feature] **llms.txt Specification Compliance** - Updated output format to fully comply with the llms.txt specification from llmstxt.org.
   - **Metadata Format**: Metadata now appears within the description field using parentheses and comma separators: `- [title](url): description (tokens:450, updated:2025-10-13, priority:high)`

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    llm-docs-builder (0.10.0)
+    llm-docs-builder (0.11.0)
       zeitwerk (~> 2.6)
 
 GEM
@@ -12,15 +12,15 @@ GEM
     coderay (1.1.3)
     diff-lcs (1.6.2)
     docile (1.4.1)
-    json (2.13.2)
+    json (2.15.2)
     language_server-protocol (3.17.0.5)
     lint_roller (1.1.0)
     method_source (1.1.0)
     parallel (1.27.0)
-    parser (3.3.9.0)
+    parser (3.3.10.0)
       ast (~> 2.4.1)
       racc
-    prism (1.4.0)
+    prism (1.6.0)
     pry (0.15.2)
       coderay (~> 1.1)
       method_source (~> 1.0)
@@ -29,22 +29,22 @@ GEM
       pry (>= 0.13, < 0.16)
     racc (1.8.1)
     rainbow (3.1.1)
-    rake (13.3.0)
-    regexp_parser (2.11.2)
-    rspec (3.13.1)
+    rake (13.3.1)
+    regexp_parser (2.11.3)
+    rspec (3.13.2)
       rspec-core (~> 3.13.0)
       rspec-expectations (~> 3.13.0)
       rspec-mocks (~> 3.13.0)
-    rspec-core (3.13.5)
+    rspec-core (3.13.6)
       rspec-support (~> 3.13.0)
     rspec-expectations (3.13.5)
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.13.0)
-    rspec-mocks (3.13.5)
+    rspec-mocks (3.13.6)
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.13.0)
-    rspec-support (3.13.5)
-    rubocop (1.80.0)
+    rspec-support (3.13.6)
+    rubocop (1.81.6)
       json (~> 2.3)
       language_server-protocol (~> 3.17.0.2)
       lint_roller (~> 1.1.0)
@@ -52,10 +52,10 @@ GEM
       parser (>= 3.3.0.2)
       rainbow (>= 2.2.2, < 4.0)
       regexp_parser (>= 2.9.3, < 3.0)
-      rubocop-ast (>= 1.46.0, < 2.0)
+      rubocop-ast (>= 1.47.1, < 2.0)
       ruby-progressbar (~> 1.7)
       unicode-display_width (>= 2.4.0, < 4.0)
-    rubocop-ast (1.46.0)
+    rubocop-ast (1.47.1)
       parser (>= 3.3.7.2)
       prism (~> 1.4)
     ruby-progressbar (1.13.0)
@@ -65,9 +65,9 @@ GEM
       simplecov_json_formatter (~> 0.1)
     simplecov-html (0.13.2)
     simplecov_json_formatter (0.1.4)
-    unicode-display_width (3.1.5)
-      unicode-emoji (~> 4.0, >= 4.0.4)
-    unicode-emoji (4.0.4)
+    unicode-display_width (3.2.0)
+      unicode-emoji (~> 4.1)
+    unicode-emoji (4.1.0)
     zeitwerk (2.7.3)
 
 PLATFORMS
@@ -85,4 +85,4 @@ DEPENDENCIES
   simplecov (~> 0.21)
 
 BUNDLED WITH
-   2.7.1
+   2.7.2

data/README.md

@@ -61,6 +61,9 @@ Factor:         2.8x smaller
 # Single file
 llm-docs-builder transform --docs README.md
 
+# Fetch and transform a remote page
+llm-docs-builder transform --url https://yoursite.com/docs/page.html
+
 # Bulk transform with config
 llm-docs-builder bulk-transform --config llm-docs-builder.yml
 ```

data/lib/llm_docs_builder/cli.rb

@@ -68,8 +68,9 @@ module LlmDocsBuilder
     # @param argv [Array<String>] command-line arguments
     # @return [Hash] parsed options including :command, :config, :docs, :output, :verbose
     def parse_options(argv)
+      command_token = argv.first
       options = {
-        command: argv.first&.match?(/^[a-z-]+$/) ? argv.shift : nil
+        command: command_token&.match?(/\A[a-z](?:[a-z-]*[a-z])?\z/) ? argv.shift : nil
       }
 
       OptionParser.new do |opts|
@@ -100,7 +101,7 @@ module LlmDocsBuilder
           options[:output] = path
         end
 
-        opts.on('-u', '--url URL', 'URL to fetch for comparison') do |url|
+        opts.on('-u', '--url URL', 'URL to fetch for transform or comparison') do |url|
           options[:url] = url
         end
 
@@ -185,21 +186,42 @@ module LlmDocsBuilder
       config = LlmDocsBuilder::Config.new(options[:config])
       merged_options = config.merge_with_options(options)
 
-      file_path = merged_options[:docs]
+      url = options[:url]
+      cli_file_path = options[:docs]
+      config_file_path = config['docs']
+      file_path = url ? cli_file_path : (cli_file_path || config_file_path)
 
-      unless file_path
-        puts 'File path required for transform command (use -d/--docs)'
+      if url && cli_file_path
+        puts 'Cannot use both --docs and --url for transform command'
         exit 1
       end
 
-      unless File.exist?(file_path)
-        puts "File not found: #{file_path}"
-        exit 1
+      unless file_path
+        unless url
+          puts 'File path required for transform command (use -d/--docs)'
+          exit 1
+        end
       end
 
-      puts "Transforming #{file_path}..." if merged_options[:verbose]
+      content =
+        if url
+          puts "Fetching #{url}..." if merged_options[:verbose]
+          fetcher = LlmDocsBuilder::UrlFetcher.new(verbose: merged_options[:verbose])
+          remote_content = fetcher.fetch(url)
+          puts "Transforming content from #{url}..." if merged_options[:verbose]
+          transform_options = merged_options.merge(content: remote_content, docs: nil, source_url: url)
+          LlmDocsBuilder.transform_markdown(nil, transform_options)
+        else
+          unless File.exist?(file_path)
+            puts "File not found: #{file_path}"
+            exit 1
+          end
 
-      content = LlmDocsBuilder.transform_markdown(file_path, merged_options)
+          puts "Transforming #{file_path}..." if merged_options[:verbose]
+
+          merged_options[:docs] = file_path
+          LlmDocsBuilder.transform_markdown(file_path, merged_options)
+        end
 
       if merged_options[:output] && merged_options[:output] != 'llms.txt'
         File.write(merged_options[:output], content)

data/lib/llm_docs_builder/comparator.rb

@@ -1,8 +1,5 @@
 # frozen_string_literal: true
 
-require 'net/http'
-require 'uri'
-
 module LlmDocsBuilder
   # Compares content sizes between human and AI versions
   #
@@ -30,7 +27,7 @@ module LlmDocsBuilder
     AI_USER_AGENT = 'Claude-Web/1.0 (Anthropic AI Assistant)'
 
     # Maximum number of redirects to follow before raising an error
-    MAX_REDIRECTS = 10
+    MAX_REDIRECTS = UrlFetcher::MAX_REDIRECTS
 
     # @return [String] URL to compare
     attr_reader :url
@@ -133,78 +130,11 @@ module LlmDocsBuilder
     # @return [String] response body
     # @raise [Errors::GenerationError] if fetch fails or too many redirects
     def fetch_url(url_string, user_agent, redirect_count = 0)
-      if redirect_count >= MAX_REDIRECTS
-        raise(
-          Errors::GenerationError,
-          "Too many redirects (#{MAX_REDIRECTS}) when fetching #{url_string}"
-        )
-      end
-
-      uri = validate_and_parse_url(url_string)
-
-      http = Net::HTTP.new(uri.host, uri.port)
-      http.use_ssl = uri.scheme == 'https'
-      http.open_timeout = 10
-      http.read_timeout = 30
-
-      request = Net::HTTP::Get.new(uri.request_uri)
-      request['User-Agent'] = user_agent
-
-      response = http.request(request)
-
-      case response
-      when Net::HTTPSuccess
-        response.body
-      when Net::HTTPRedirection
-        # Follow redirect with incremented counter
-        redirect_url = response['location']
-        puts "  Redirecting to #{redirect_url}..." if options[:verbose] && redirect_count.positive?
-        fetch_url(redirect_url, user_agent, redirect_count + 1)
-      else
-        raise(
-          Errors::GenerationError,
-          "Failed to fetch #{url_string}: #{response.code} #{response.message}"
-        )
-      end
-    rescue Errors::GenerationError
-      raise
-    rescue StandardError => e
-      raise(
-        Errors::GenerationError,
-        "Error fetching #{url_string}: #{e.message}"
-      )
-    end
-
-    # Validates and parses URL to prevent malformed URLs
-    #
-    # @param url_string [String] URL to validate and parse
-    # @return [URI::HTTP, URI::HTTPS] parsed URI
-    # @raise [Errors::GenerationError] if URL is invalid or uses unsupported scheme
-    def validate_and_parse_url(url_string)
-      uri = URI.parse(url_string)
-
-      # Only allow HTTP and HTTPS schemes
-      unless %w[http https].include?(uri.scheme&.downcase)
-        raise(
-          Errors::GenerationError,
-          "Unsupported URL scheme: #{uri.scheme || 'none'} (only http/https allowed)"
-        )
-      end
-
-      # Ensure host is present
-      if uri.host.nil? || uri.host.empty?
-        raise(
-          Errors::GenerationError,
-          "Invalid URL: missing host in #{url_string}"
-        )
-      end
-
-      uri
-    rescue URI::InvalidURIError => e
-      raise(
-        Errors::GenerationError,
-        "Invalid URL format: #{e.message}"
+      fetcher = UrlFetcher.new(
+        user_agent: user_agent,
+        verbose: options[:verbose]
       )
+      fetcher.fetch(url_string, redirect_count)
     end
 
     # Calculate comparison statistics

data/lib/llm_docs_builder/config.rb

@@ -57,7 +57,11 @@ module LlmDocsBuilder
     def merge_with_options(options)
       # CLI options override config file, config file provides defaults
       {
-        docs: options[:docs] || self['docs'] || '.',
+        docs: if options.key?(:docs)
+                options[:docs]
+              else
+                self['docs'] || '.'
+              end,
         base_url: options[:base_url] || self['base_url'],
         title: options[:title] || self['title'],
         description: options[:description] || self['description'],
@@ -171,7 +175,10 @@ module LlmDocsBuilder
                                else
                                  self['calculate_compression'] || false
                                end
-      }
+      }.tap do |merged|
+        merged[:content] = options[:content] if options.key?(:content)
+        merged[:source_url] = options[:source_url] if options.key?(:source_url)
+      end
     end
 
     # Check if a config file was found and exists

data/lib/llm_docs_builder/markdown_transformer.rb

@@ -55,7 +55,7 @@ module LlmDocsBuilder
     #
     # @return [String] transformed markdown content
     def transform
-      content = File.read(file_path)
+      content = load_content
 
       # Build and execute transformation pipeline
       content = cleanup_transformer.transform(content, options)
@@ -124,5 +124,16 @@ module LlmDocsBuilder
       }
       compressor.compress(content, compression_methods)
     end
+
+    # Load source content either from provided string or file path
+    #
+    # @return [String] markdown content to transform
+    def load_content
+      if options[:content]
+        options[:content].dup
+      else
+        File.read(file_path)
+      end
+    end
   end
 end

data/lib/llm_docs_builder/url_fetcher.rb

@@ -0,0 +1,120 @@
+# frozen_string_literal: true
+
+require 'net/http'
+require 'uri'
+
+module LlmDocsBuilder
+  # Lightweight HTTP client for fetching remote documentation pages.
+  #
+  # Provides common functionality needed by multiple commands (transform, compare)
+  # including strict scheme validation, redirect handling and sensible timeouts.
+  class UrlFetcher
+    DEFAULT_USER_AGENT = 'llm-docs-builder/1.0 (+https://github.com/mensfeld/llm-docs-builder)'
+    MAX_REDIRECTS = 10
+
+    # @param user_agent [String] HTTP user agent header value
+    # @param verbose [Boolean] enable redirect logging
+    # @param output [IO] IO stream used for redirect logging
+    def initialize(user_agent: DEFAULT_USER_AGENT, verbose: false, output: $stdout)
+      @user_agent = user_agent
+      @verbose = verbose
+      @output = output
+    end
+
+    # Fetch remote URL content while following redirects.
+    #
+    # @param url_string [String] URL to fetch
+    # @param redirect_count [Integer] current redirect depth (internal use)
+    # @return [String] response body
+    # @raise [Errors::GenerationError] on invalid URLs, network failures, or redirect loops
+    def fetch(url_string, redirect_count = 0)
+      if redirect_count >= MAX_REDIRECTS
+        raise(
+          Errors::GenerationError,
+          "Too many redirects (#{MAX_REDIRECTS}) when fetching #{url_string}"
+        )
+      end
+
+      uri = validate_and_parse_url(url_string)
+
+      http = Net::HTTP.new(uri.host, uri.port)
+      http.use_ssl = uri.scheme == 'https'
+      http.open_timeout = 10
+      http.read_timeout = 30
+
+      request = Net::HTTP::Get.new(uri.request_uri)
+      request['User-Agent'] = @user_agent
+
+      response = http.request(request)
+
+      case response
+      when Net::HTTPSuccess
+        response.body
+      when Net::HTTPRedirection
+        redirect_url = absolute_redirect_url(uri, response['location'])
+        log_redirect(redirect_url)
+        fetch(redirect_url, redirect_count + 1)
+      else
+        raise(
+          Errors::GenerationError,
+          "Failed to fetch #{url_string}: #{response.code} #{response.message}"
+        )
+      end
+    rescue Errors::GenerationError
+      raise
+    rescue StandardError => e
+      raise(
+        Errors::GenerationError,
+        "Error fetching #{url_string}: #{e.message}"
+      )
+    end
+
+    private
+
+    def validate_and_parse_url(url_string)
+      uri = URI.parse(url_string)
+
+      unless %w[http https].include?(uri.scheme&.downcase)
+        raise(
+          Errors::GenerationError,
+          "Unsupported URL scheme: #{uri.scheme || 'none'} (only http/https allowed)"
+        )
+      end
+
+      if uri.host.nil? || uri.host.empty?
+        raise(
+          Errors::GenerationError,
+          "Invalid URL: missing host in #{url_string}"
+        )
+      end
+
+      uri
+    rescue URI::InvalidURIError => e
+      raise(
+        Errors::GenerationError,
+        "Invalid URL format: #{e.message}"
+      )
+    end
+
+    def absolute_redirect_url(base_uri, location)
+      raise(
+        Errors::GenerationError,
+        "Redirect missing location header for #{base_uri}"
+      ) if location.nil? || location.empty?
+
+      URI.join(base_uri, location).to_s
+    rescue URI::InvalidURIError => e
+      raise(
+        Errors::GenerationError,
+        "Invalid redirect URL from #{base_uri}: #{e.message}"
+      )
+    end
+
+    def log_redirect(url)
+      return unless @verbose
+
+      @output.puts("  Redirecting to #{url}...")
+    end
+  end
+end
+

data/lib/llm_docs_builder/version.rb

@@ -2,5 +2,5 @@
 
 module LlmDocsBuilder
   # Current version of the LlmDocsBuilder gem
-  VERSION = '0.10.0'
+  VERSION = '0.11.0'
 end

data/lib/llm_docs_builder.rb

@@ -25,6 +25,7 @@ module LlmDocsBuilder
     # @option options [Boolean] :convert_urls convert HTML URLs to markdown format (overrides
     #   config)
     # @option options [Boolean] :verbose enable verbose output (overrides config)
+    # @option options [String] :content raw markdown content (used for remote sources)
     # @return [String] generated llms.txt content
     #
     # @example Generate from docs directory

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: llm-docs-builder
 version: !ruby/object:Gem::Version
-  version: 0.10.0
+  version: 0.11.0
 platform: ruby
 authors:
 - Maciej Mensfeld
@@ -116,6 +116,7 @@ files:
 - ".rspec"
 - ".rubocop.yml"
 - ".ruby-version"
+- AGENTS.md
 - CHANGELOG.md
 - Dockerfile
 - Gemfile
@@ -143,6 +144,7 @@ files:
 - lib/llm_docs_builder/transformers/heading_transformer.rb
 - lib/llm_docs_builder/transformers/link_transformer.rb
 - lib/llm_docs_builder/transformers/whitespace_transformer.rb
+- lib/llm_docs_builder/url_fetcher.rb
 - lib/llm_docs_builder/validator.rb
 - lib/llm_docs_builder/version.rb
 - llm-docs-builder.gemspec