llm-docs-builder 0.10.0 → 0.12.0

data/lib/llm_docs_builder/markdown_transformer.rb

@@ -55,7 +55,7 @@ module LlmDocsBuilder
     #
     # @return [String] transformed markdown content
     def transform
-      content = File.read(file_path)
+      content = load_content
 
       # Build and execute transformation pipeline
       content = cleanup_transformer.transform(content, options)
@@ -63,9 +63,7 @@ module LlmDocsBuilder
       content = heading_transformer.transform(content, options)
       content = compress_content(content) if should_compress?
       content = enhancement_transformer.transform(content, options)
-      content = whitespace_transformer.transform(content, options)
-
-      content
+      whitespace_transformer.transform(content, options)
     end
 
     private
@@ -114,7 +112,7 @@ module LlmDocsBuilder
 
     # Compress content using TextCompressor
     #
-    # @param content [String] content to compress
+    # @param content [String] markdown content
     # @return [String] compressed content
     def compress_content(content)
       compressor = TextCompressor.new
@@ -124,5 +122,32 @@ module LlmDocsBuilder
       }
       compressor.compress(content, compression_methods)
     end
+
+    # Load source content either from provided string or file path
+    #
+    # @return [String] markdown content to transform
+    def load_content
+      content = options[:content] ? options[:content].dup : File.read(file_path)
+      snippet = html_detector.detection_snippet(content)
+
+      return content if html_detector.table_fragment?(snippet)
+      return html_to_markdown_converter.convert(content) if html_detector.html_content?(content, snippet)
+
+      content
+    end
+
+    # Memoized HTML to markdown converter
+    #
+    # @return [HtmlToMarkdownConverter]
+    def html_to_markdown_converter
+      @html_to_markdown_converter ||= HtmlToMarkdownConverter.new
+    end
+
+    # Memoized HTML detector
+    #
+    # @return [HtmlDetector]
+    def html_detector
+      @html_detector ||= HtmlDetector.new
+    end
   end
 end

data/lib/llm_docs_builder/output_formatter.rb

@@ -30,7 +30,7 @@ module LlmDocsBuilder
 
     # Format number with comma separators for readability
     #
-    # @param number [Integer] number to format
+    # @param number [Integer] integer value
     # @return [String] formatted number with commas
     #
     # @example

data/lib/llm_docs_builder/transformers/base_transformer.rb

@@ -1,6 +1,13 @@
 # frozen_string_literal: true
 
 module LlmDocsBuilder
+  # Provides content transformation functionality
+  #
+  # This module contains specialized transformers for modifying markdown content,
+  # including cleanup operations, link processing, heading normalization, and
+  # content enhancement for AI consumption.
+  #
+  # @api private
   module Transformers
     # Base module for all transformers
     #
@@ -11,9 +18,12 @@ module LlmDocsBuilder
     module BaseTransformer
       # Transform content
       #
-      # @param content [String] content to transform
+      # @abstract Subclasses must implement this method and document specific options
+      # @param content [String] markdown content
       # @param options [Hash] transformation options
+      # @option options [Object] :* options vary by implementation - see specific transformer classes
       # @return [String] transformed content
+      # @note Options vary by implementation - see specific transformer classes for supported keys
       def transform(content, options = {})
         raise NotImplementedError, "#{self.class} must implement #transform"
       end
@@ -21,7 +31,9 @@ module LlmDocsBuilder
       # Check if transformation should be applied
       #
       # @param options [Hash] transformation options
+      # @option options [Object] :* options vary by implementation - see specific transformer classes
       # @return [Boolean] true if transformation should be applied
+      # @note Options vary by implementation - see specific transformer classes for supported keys
       def should_transform?(options)
         true
       end

data/lib/llm_docs_builder/url_fetcher.rb

@@ -0,0 +1,138 @@
+# frozen_string_literal: true
+
+require 'net/http'
+require 'uri'
+
+module LlmDocsBuilder
+  # Lightweight HTTP client for fetching remote documentation pages.
+  #
+  # Provides common functionality needed by multiple commands (transform, compare)
+  # including strict scheme validation, redirect handling and sensible timeouts.
+  class UrlFetcher
+    # Default user agent string for HTTP requests
+    DEFAULT_USER_AGENT = 'llm-docs-builder/1.0 (+https://github.com/mensfeld/llm-docs-builder)'
+
+    # Maximum number of redirects to follow
+    MAX_REDIRECTS = 10
+
+    # @param user_agent [String] HTTP user agent header value
+    # @param verbose [Boolean] enable redirect logging
+    # @param output [IO] IO stream used for redirect logging
+    def initialize(user_agent: DEFAULT_USER_AGENT, verbose: false, output: $stdout)
+      @user_agent = user_agent
+      @verbose = verbose
+      @output = output
+    end
+
+    # Fetch remote URL content while following redirects.
+    #
+    # @param url_string [String] URL to fetch
+    # @param redirect_count [Integer] current redirect depth (internal use)
+    # @return [String] response body
+    # @raise [Errors::GenerationError] on invalid URLs, network failures, or redirect loops
+    def fetch(url_string, redirect_count = 0)
+      if redirect_count >= MAX_REDIRECTS
+        raise(
+          Errors::GenerationError,
+          "Too many redirects (#{MAX_REDIRECTS}) when fetching #{url_string}"
+        )
+      end
+
+      uri = validate_and_parse_url(url_string)
+
+      http = Net::HTTP.new(uri.host, uri.port)
+      http.use_ssl = uri.scheme == 'https'
+      http.open_timeout = 10
+      http.read_timeout = 30
+
+      request = Net::HTTP::Get.new(uri.request_uri)
+      request['User-Agent'] = @user_agent
+
+      response = http.request(request)
+
+      case response
+      when Net::HTTPSuccess
+        response.body
+      when Net::HTTPRedirection
+        redirect_url = absolute_redirect_url(uri, response['location'])
+        log_redirect(redirect_url)
+        fetch(redirect_url, redirect_count + 1)
+      else
+        raise(
+          Errors::GenerationError,
+          "Failed to fetch #{url_string}: #{response.code} #{response.message}"
+        )
+      end
+    rescue Errors::GenerationError
+      raise
+    rescue StandardError => e
+      raise(
+        Errors::GenerationError,
+        "Error fetching #{url_string}: #{e.message}"
+      )
+    end
+
+    private
+
+    # Validate and parse URL string
+    #
+    # @param url_string [String] URL to validate
+    # @return [URI::HTTP, URI::HTTPS] parsed URI
+    # @raise [Errors::GenerationError] if URL is invalid or unsupported
+    def validate_and_parse_url(url_string)
+      uri = URI.parse(url_string)
+
+      unless %w[http https].include?(uri.scheme&.downcase)
+        raise(
+          Errors::GenerationError,
+          "Unsupported URL scheme: #{uri.scheme || 'none'} (only http/https allowed)"
+        )
+      end
+
+      if uri.host.nil? || uri.host.empty?
+        raise(
+          Errors::GenerationError,
+          "Invalid URL: missing host in #{url_string}"
+        )
+      end
+
+      uri
+    rescue URI::InvalidURIError => e
+      raise(
+        Errors::GenerationError,
+        "Invalid URL format: #{e.message}"
+      )
+    end
+
+    # Convert redirect location to absolute URL
+    #
+    # @param base_uri [URI] base URI
+    # @param location [String] redirect location
+    # @return [String] absolute redirect URL
+    # @raise [Errors::GenerationError] if location is invalid
+    def absolute_redirect_url(base_uri, location)
+      raise(
+        Errors::GenerationError,
+        "Redirect missing location header for #{base_uri}"
+      ) if location.nil? || location.empty?
+
+      URI.join(base_uri, location).to_s
+    rescue URI::InvalidURIError => e
+      raise(
+        Errors::GenerationError,
+        "Invalid redirect URL from #{base_uri}: #{e.message}"
+      )
+    end
+
+    # Log redirect if verbose mode enabled
+    #
+    # @param url [String] redirect URL
+    # @return [void]
+    def log_redirect(url)
+      return unless @verbose
+
+      @output.puts("  Redirecting to #{url}...")
+    end
+  end
+end
+

data/lib/llm_docs_builder/version.rb

@@ -2,5 +2,5 @@
 
 module LlmDocsBuilder
   # Current version of the LlmDocsBuilder gem
-  VERSION = '0.10.0'
+  VERSION = '0.12.0'
 end

data/lib/llm_docs_builder.rb

@@ -4,10 +4,20 @@ require 'zeitwerk'
 require 'pathname'
 require 'find'
 
+autoload(:Nokogiri, 'nokogiri')
+autoload(:CGI, 'cgi')
+
 loader = Zeitwerk::Loader.for_gem
 loader.inflector.inflect('cli' => 'CLI')
 loader.setup
 
+# Build and optimize documentation for LLMs
+#
+# This gem provides tools for generating llms.txt files and transforming markdown
+# documentation to be AI-friendly. It can reduce token consumption by 67-95% while
+# preserving essential documentation content.
+#
+# @api public
 module LlmDocsBuilder
   class << self
     # Generates llms.txt from existing markdown documentation
@@ -25,6 +35,7 @@ module LlmDocsBuilder
     # @option options [Boolean] :convert_urls convert HTML URLs to markdown format (overrides
     #   config)
     # @option options [Boolean] :verbose enable verbose output (overrides config)
+    # @option options [String] :content raw markdown content (used for remote sources)
     # @return [String] generated llms.txt content
     #
     # @example Generate from docs directory

data/llm-docs-builder.gemspec

@@ -35,6 +35,7 @@ Gem::Specification.new do |spec|
   spec.executables = spec.files.grep(%r{\Abin/}) { |f| File.basename(f) }
   spec.require_paths = ['lib']
 
+  spec.add_dependency 'nokogiri', '~> 1.17'
   spec.add_dependency 'zeitwerk', '~> 2.6'
 
   spec.add_development_dependency 'bundler', '~> 2.0'

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: llm-docs-builder
 version: !ruby/object:Gem::Version
-  version: 0.10.0
+  version: 0.12.0
 platform: ruby
 authors:
 - Maciej Mensfeld
@@ -9,6 +9,20 @@ bindir: bin
 cert_chain: []
 date: 1980-01-02 00:00:00.000000000 Z
 dependencies:
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.17'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.17'
 - !ruby/object:Gem::Dependency
   name: zeitwerk
   requirement: !ruby/object:Gem::Requirement
@@ -132,6 +146,13 @@ files:
 - lib/llm_docs_builder/config.rb
 - lib/llm_docs_builder/errors.rb
 - lib/llm_docs_builder/generator.rb
+- lib/llm_docs_builder/helpers.rb
+- lib/llm_docs_builder/helpers/prune_trailing_unsafe_link_separator.rb
+- lib/llm_docs_builder/helpers/squeeze_blank_lines_outside_fences.rb
+- lib/llm_docs_builder/html_detector.rb
+- lib/llm_docs_builder/html_to_markdown/figure_code_block_renderer.rb
+- lib/llm_docs_builder/html_to_markdown/table_markup_renderer.rb
+- lib/llm_docs_builder/html_to_markdown_converter.rb
 - lib/llm_docs_builder/markdown_transformer.rb
 - lib/llm_docs_builder/output_formatter.rb
 - lib/llm_docs_builder/parser.rb
@@ -143,6 +164,7 @@ files:
 - lib/llm_docs_builder/transformers/heading_transformer.rb
 - lib/llm_docs_builder/transformers/link_transformer.rb
 - lib/llm_docs_builder/transformers/whitespace_transformer.rb
+- lib/llm_docs_builder/url_fetcher.rb
 - lib/llm_docs_builder/validator.rb
 - lib/llm_docs_builder/version.rb
 - llm-docs-builder.gemspec