llm-docs-builder 0.11.0 → 0.12.0

data/lib/llm_docs_builder/markdown_transformer.rb

@@ -63,9 +63,7 @@ module LlmDocsBuilder
       content = heading_transformer.transform(content, options)
       content = compress_content(content) if should_compress?
       content = enhancement_transformer.transform(content, options)
-      content = whitespace_transformer.transform(content, options)
-
-      content
+      whitespace_transformer.transform(content, options)
     end
 
     private
@@ -114,7 +112,7 @@ module LlmDocsBuilder
 
     # Compress content using TextCompressor
     #
-    # @param content [String] content to compress
+    # @param content [String] markdown content
     # @return [String] compressed content
     def compress_content(content)
       compressor = TextCompressor.new
@@ -129,11 +127,27 @@ module LlmDocsBuilder
     #
     # @return [String] markdown content to transform
     def load_content
-      if options[:content]
-        options[:content].dup
-      else
-        File.read(file_path)
-      end
+      content = options[:content] ? options[:content].dup : File.read(file_path)
+      snippet = html_detector.detection_snippet(content)
+
+      return content if html_detector.table_fragment?(snippet)
+      return html_to_markdown_converter.convert(content) if html_detector.html_content?(content, snippet)
+
+      content
+    end
+
+    # Memoized HTML to markdown converter
+    #
+    # @return [HtmlToMarkdownConverter]
+    def html_to_markdown_converter
+      @html_to_markdown_converter ||= HtmlToMarkdownConverter.new
+    end
+
+    # Memoized HTML detector
+    #
+    # @return [HtmlDetector]
+    def html_detector
+      @html_detector ||= HtmlDetector.new
     end
   end
 end

data/lib/llm_docs_builder/output_formatter.rb

@@ -30,7 +30,7 @@ module LlmDocsBuilder
 
     # Format number with comma separators for readability
     #
-    # @param number [Integer] number to format
+    # @param number [Integer] integer value
     # @return [String] formatted number with commas
     #
     # @example

data/lib/llm_docs_builder/transformers/base_transformer.rb

@@ -1,6 +1,13 @@
 # frozen_string_literal: true
 
 module LlmDocsBuilder
+  # Provides content transformation functionality
+  #
+  # This module contains specialized transformers for modifying markdown content,
+  # including cleanup operations, link processing, heading normalization, and
+  # content enhancement for AI consumption.
+  #
+  # @api private
   module Transformers
     # Base module for all transformers
     #
@@ -11,9 +18,12 @@ module LlmDocsBuilder
     module BaseTransformer
       # Transform content
       #
-      # @param content [String] content to transform
+      # @abstract Subclasses must implement this method and document specific options
+      # @param content [String] markdown content
       # @param options [Hash] transformation options
+      # @option options [Object] :* options vary by implementation - see specific transformer classes
       # @return [String] transformed content
+      # @note Options vary by implementation - see specific transformer classes for supported keys
       def transform(content, options = {})
         raise NotImplementedError, "#{self.class} must implement #transform"
       end
@@ -21,7 +31,9 @@ module LlmDocsBuilder
       # Check if transformation should be applied
       #
       # @param options [Hash] transformation options
+      # @option options [Object] :* options vary by implementation - see specific transformer classes
       # @return [Boolean] true if transformation should be applied
+      # @note Options vary by implementation - see specific transformer classes for supported keys
       def should_transform?(options)
         true
       end

data/lib/llm_docs_builder/url_fetcher.rb

@@ -9,7 +9,10 @@ module LlmDocsBuilder
   # Provides common functionality needed by multiple commands (transform, compare)
   # including strict scheme validation, redirect handling and sensible timeouts.
   class UrlFetcher
+    # Default user agent string for HTTP requests
     DEFAULT_USER_AGENT = 'llm-docs-builder/1.0 (+https://github.com/mensfeld/llm-docs-builder)'
+
+    # Maximum number of redirects to follow
     MAX_REDIRECTS = 10
 
     # @param user_agent [String] HTTP user agent header value
@@ -71,6 +74,11 @@ module LlmDocsBuilder
 
     private
 
+    # Validate and parse URL string
+    #
+    # @param url_string [String] URL to validate
+    # @return [URI::HTTP, URI::HTTPS] parsed URI
+    # @raise [Errors::GenerationError] if URL is invalid or unsupported
     def validate_and_parse_url(url_string)
       uri = URI.parse(url_string)
 
@@ -96,6 +104,12 @@ module LlmDocsBuilder
       )
     end
 
+    # Convert redirect location to absolute URL
+    #
+    # @param base_uri [URI] base URI
+    # @param location [String] redirect location
+    # @return [String] absolute redirect URL
+    # @raise [Errors::GenerationError] if location is invalid
     def absolute_redirect_url(base_uri, location)
       raise(
         Errors::GenerationError,
@@ -110,6 +124,10 @@ module LlmDocsBuilder
       )
     end
 
+    # Log redirect if verbose mode enabled
+    #
+    # @param url [String] redirect URL
+    # @return [void]
     def log_redirect(url)
       return unless @verbose
 

data/lib/llm_docs_builder/version.rb

@@ -2,5 +2,5 @@
 
 module LlmDocsBuilder
   # Current version of the LlmDocsBuilder gem
-  VERSION = '0.11.0'
+  VERSION = '0.12.0'
 end

data/lib/llm_docs_builder.rb

@@ -4,10 +4,20 @@ require 'zeitwerk'
 require 'pathname'
 require 'find'
 
+autoload(:Nokogiri, 'nokogiri')
+autoload(:CGI, 'cgi')
+
 loader = Zeitwerk::Loader.for_gem
 loader.inflector.inflect('cli' => 'CLI')
 loader.setup
 
+# Build and optimize documentation for LLMs
+#
+# This gem provides tools for generating llms.txt files and transforming markdown
+# documentation to be AI-friendly. It can reduce token consumption by 67-95% while
+# preserving essential documentation content.
+#
+# @api public
 module LlmDocsBuilder
   class << self
     # Generates llms.txt from existing markdown documentation

data/llm-docs-builder.gemspec

@@ -35,6 +35,7 @@ Gem::Specification.new do |spec|
   spec.executables = spec.files.grep(%r{\Abin/}) { |f| File.basename(f) }
   spec.require_paths = ['lib']
 
+  spec.add_dependency 'nokogiri', '~> 1.17'
   spec.add_dependency 'zeitwerk', '~> 2.6'
 
   spec.add_development_dependency 'bundler', '~> 2.0'

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: llm-docs-builder
 version: !ruby/object:Gem::Version
-  version: 0.11.0
+  version: 0.12.0
 platform: ruby
 authors:
 - Maciej Mensfeld
@@ -9,6 +9,20 @@ bindir: bin
 cert_chain: []
 date: 1980-01-02 00:00:00.000000000 Z
 dependencies:
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.17'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.17'
 - !ruby/object:Gem::Dependency
   name: zeitwerk
   requirement: !ruby/object:Gem::Requirement
@@ -116,7 +130,6 @@ files:
 - ".rspec"
 - ".rubocop.yml"
 - ".ruby-version"
-- AGENTS.md
 - CHANGELOG.md
 - Dockerfile
 - Gemfile
@@ -133,6 +146,13 @@ files:
 - lib/llm_docs_builder/config.rb
 - lib/llm_docs_builder/errors.rb
 - lib/llm_docs_builder/generator.rb
+- lib/llm_docs_builder/helpers.rb
+- lib/llm_docs_builder/helpers/prune_trailing_unsafe_link_separator.rb
+- lib/llm_docs_builder/helpers/squeeze_blank_lines_outside_fences.rb
+- lib/llm_docs_builder/html_detector.rb
+- lib/llm_docs_builder/html_to_markdown/figure_code_block_renderer.rb
+- lib/llm_docs_builder/html_to_markdown/table_markup_renderer.rb
+- lib/llm_docs_builder/html_to_markdown_converter.rb
 - lib/llm_docs_builder/markdown_transformer.rb
 - lib/llm_docs_builder/output_formatter.rb
 - lib/llm_docs_builder/parser.rb

data/AGENTS.md

@@ -1,20 +0,0 @@
-# Repository Guidelines
-
-## Project Structure & Module Organization
-Core gem code lives in `lib/llm_docs_builder`, with single-responsibility modules such as `generator.rb`, `validator.rb`, and the CLI glue in `cli.rb`. Shared entrypoint `lib/llm_docs_builder.rb` wires dependencies. Executables reside in `bin/`: `llm-docs-builder` boots the CLI, while `rspecs` runs the full test matrix. Specs mirror library files under `spec/` with command-level coverage in `spec/integrations`. Static assets (logos, diff screenshots) are in `misc/`. Example configuration templates live at `llm-docs-builder.yml.example`.
-
-## Build, Test, and Development Commands
-- `bundle install` — sync gem dependencies defined in `Gemfile`.
-- `bundle exec rake` — default task; runs RSpec and RuboCop together.
-- `bundle exec rspec` or `bin/rspecs` — execute unit and integration specs with doc formatter.
-- `bundle exec rubocop` — enforce the Ruby style guide; mirrors CI.
-- `bin/llm-docs-builder transform --docs README.md` — smoke-test the CLI against a local file.
-
-## Coding Style & Naming Conventions
-Target Ruby 3.2 with two-space indentation and trailing newline. Prefer single-quoted strings; enable `# frozen_string_literal: true` headers on Ruby files. Keep lines ≤120 characters except where the RuboCop config allows. Use descriptive module/class names (e.g., `LlmDocsBuilder::Generator`) and predicate methods ending with `?` when returning booleans. Place supporting fixtures in `spec/support` if added, and name files after the class they extend.
-
-## Testing Guidelines
-RSpec is the sole testing framework. Name files `*_spec.rb` and align describe blocks with constant paths. Integration scenarios belong in `spec/integrations` to capture CLI behaviors. SimpleCov is enabled by default for line and branch coverage; export `SIMPLECOV=false` for quick local runs. Persist example statuses with the automatically managed `spec/examples.txt`.
-
-## Commit & Pull Request Guidelines
-Keep commit subjects short, present-tense, and focused (e.g., `Align CLI config (#27)`). Group related changes together so `git log` remains readable. Pull requests should describe motivation, summarize behavioral impact, link related issues or discussions, and include CLI output or screenshots when touching generated docs. Ensure CI passes (`bundle exec rake`) before requesting review, and note any follow-up work in the PR description.