llm-docs-builder 0.12.0 → 1.0.0

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    llm-docs-builder (0.12.0)
+    llm-docs-builder (1.0.0)
       nokogiri (~> 1.17)
       zeitwerk (~> 2.6)
 
@@ -9,49 +9,31 @@ GEM
   remote: https://rubygems.org/
   specs:
     ast (2.4.3)
-    byebug (12.0.0)
     coderay (1.1.3)
-    date (3.5.0)
     diff-lcs (1.6.2)
-    docile (1.4.1)
-    erb (5.1.3)
-    io-console (0.8.1)
-    irb (1.15.3)
-      pp (>= 0.6.0)
-      rdoc (>= 4.0.0)
-      reline (>= 0.4.2)
-    json (2.16.0)
-    language_server-protocol (3.17.0.5)
+    io-console (0.9.2)
+    json (2.21.2)
+    language_server-protocol (3.17.0.6)
     lint_roller (1.1.0)
     method_source (1.1.0)
-    nokogiri (1.18.10-x86_64-linux-gnu)
+    mini_portile2 (2.8.9)
+    nokogiri (1.19.4)
+      mini_portile2 (~> 2.8.2)
       racc (~> 1.4)
-    parallel (1.27.0)
-    parser (3.3.10.0)
+    parallel (2.1.0)
+    parser (3.3.12.0)
       ast (~> 2.4.1)
       racc
-    pp (0.6.3)
-      prettyprint
-    prettyprint (0.2.0)
-    prism (1.6.0)
-    pry (0.15.2)
+    prism (1.9.0)
+    pry (0.16.0)
       coderay (~> 1.1)
       method_source (~> 1.0)
-    pry-byebug (3.11.0)
-      byebug (~> 12.0)
-      pry (>= 0.13, < 0.16)
-    psych (5.2.6)
-      date
-      stringio
+      reline (>= 0.6.0)
     racc (1.8.1)
     rainbow (3.1.1)
-    rake (13.3.1)
-    rdoc (6.15.1)
-      erb
-      psych (>= 4.0.0)
-      tsort
-    regexp_parser (2.11.3)
-    reline (0.6.3)
+    rake (13.4.2)
+    regexp_parser (2.12.0)
+    reline (0.7.0)
       io-console (~> 0.5)
     rspec (3.13.2)
       rspec-core (~> 3.13.0)
@@ -62,56 +44,49 @@ GEM
     rspec-expectations (3.13.5)
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.13.0)
-    rspec-mocks (3.13.7)
+    rspec-mocks (3.13.8)
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.13.0)
-    rspec-support (3.13.6)
-    rubocop (1.81.7)
+    rspec-support (3.13.7)
+    rubocop (1.89.0)
       json (~> 2.3)
       language_server-protocol (~> 3.17.0.2)
       lint_roller (~> 1.1.0)
-      parallel (~> 1.10)
+      parallel (>= 1.10)
       parser (>= 3.3.0.2)
       rainbow (>= 2.2.2, < 4.0)
       regexp_parser (>= 2.9.3, < 3.0)
-      rubocop-ast (>= 1.47.1, < 2.0)
+      rubocop-ast (>= 1.49.0, < 2.0)
       ruby-progressbar (~> 1.7)
       unicode-display_width (>= 2.4.0, < 4.0)
-    rubocop-ast (1.47.1)
+    rubocop-ast (1.50.0)
       parser (>= 3.3.7.2)
-      prism (~> 1.4)
+      prism (~> 1.7)
     ruby-progressbar (1.13.0)
-    simplecov (0.22.0)
-      docile (~> 1.1)
-      simplecov-html (~> 0.11)
-      simplecov_json_formatter (~> 0.1)
-    simplecov-html (0.13.2)
-    simplecov_json_formatter (0.1.4)
-    stringio (3.1.8)
-    tsort (0.2.0)
+    simplecov (1.1.1)
     unicode-display_width (3.2.0)
       unicode-emoji (~> 4.1)
-    unicode-emoji (4.1.0)
-    yard (0.9.37)
-    yard-lint (1.1.0)
-      irb
+    unicode-emoji (4.2.0)
+    warning (1.6.0)
+    yard (0.9.45)
+    yard-lint (1.11.0)
       yard (~> 0.9)
       zeitwerk (~> 2.6)
-    zeitwerk (2.7.3)
+    zeitwerk (2.8.3)
 
 PLATFORMS
-  x86_64-linux
+  ruby
 
 DEPENDENCIES
-  bundler (~> 2.0)
+  bundler
   llm-docs-builder!
   pry
-  pry-byebug
   rake (~> 13.0)
   rspec (~> 3.0)
   rubocop (~> 1.0)
-  simplecov (~> 0.21)
+  simplecov (~> 1.0)
+  warning
   yard-lint
 
 BUNDLED WITH
-   2.7.2
+  4.0.17

data/lib/llm_docs_builder/cli.rb

@@ -129,8 +129,7 @@ module LlmDocsBuilder
 
     # Generate llms.txt from documentation directory or file
     #
-    # Loads configuration, merges with CLI options, generates llms.txt content,
-    # and optionally validates the output.
+    # Loads configuration, merges with CLI options, generates llms.txt content, and optionally validates the output.
     #
     # @param options [Hash] command options from parse_options
     # @option options [String] :config path to config file
@@ -399,7 +398,6 @@ module LlmDocsBuilder
     end
 
     # Display version information
-    #
     def show_version
       puts "llm-docs-builder version #{LlmDocsBuilder::VERSION}"
     end

data/lib/llm_docs_builder/generator.rb

@@ -34,8 +34,7 @@ module LlmDocsBuilder
 
     # Generate llms.txt content from documentation
     #
-    # Scans documentation files, extracts metadata, prioritizes them, and builds a formatted
-    # llms.txt file.
+    # Scans documentation files, extracts metadata, prioritizes them, and builds a formatted llms.txt file.
     #
     # @return [String] generated llms.txt content
     def generate

data/lib/llm_docs_builder/html_detector.rb

@@ -14,8 +14,7 @@ module LlmDocsBuilder
       full_html_document?(content)
     end
 
-    # Prepare a snippet of content for HTML detection by removing leading whitespace
-    # and build metadata comments.
+    # Prepare a snippet of content for HTML detection by removing leading whitespace and build metadata comments.
     #
     # @param content [String]
     # @return [String, nil]

data/lib/llm_docs_builder/html_to_markdown/figure_code_block_renderer.rb

@@ -3,9 +3,8 @@
 module LlmDocsBuilder
   # Provides HTML to Markdown conversion functionality
   #
-  # This module contains specialized renderers for converting HTML elements
-  # to Markdown format, with support for complex structures like tables,
-  # figures, and syntax-highlighted code blocks.
+  # This module contains specialized renderers for converting HTML elements to Markdown format, with support for complex
+  # structures like tables, figures, and syntax-highlighted code blocks.
   #
   # @api private
   module HtmlToMarkdown

data/lib/llm_docs_builder/html_to_markdown_converter.rb

@@ -120,7 +120,8 @@ module LlmDocsBuilder
         text = collapsed_inline_for(element)
         return '' if text.empty?
 
-        "#{'#' * HEADING_LEVEL[tag]} #{text}"
+        effective_level = effective_heading_level(element, HEADING_LEVEL[tag])
+        "#{'#' * effective_level} #{text}"
       when 'blockquote'
         render_blockquote(element)
       when 'pre'
@@ -641,6 +642,37 @@ module LlmDocsBuilder
       out.join("\n\n")
     end
 
+    # Compute effective heading level adjusted for section nesting
+    #
+    # When HTML uses nested <section> elements with same-level headings, the inner headings should receive deeper
+    # markdown levels. The offset is calculated as the difference between the actual section ancestor count and the
+    # expected count for that heading tag (h1 expects 0 sections, h2 expects 1, etc.), capped at heading level 6.
+    #
+    # @param element [Nokogiri::XML::Element] heading element
+    # @param base_level [Integer] HTML heading level (1-6)
+    # @return [Integer] effective markdown heading level (1-6)
+    def effective_heading_level(element, base_level)
+      depth = section_ancestor_count(element)
+      offset = [depth - (base_level - 1), 0].max
+      [base_level + offset, 6].min
+    end
+
+    # Count the number of <section> ancestor elements
+    #
+    # @param element [Nokogiri::XML::Element]
+    # @return [Integer] number of section ancestors
+    def section_ancestor_count(element)
+      count = 0
+      node = element.parent
+
+      while node
+        count += 1 if node.element? && node.name.downcase == 'section'
+        node = node.parent
+      end
+
+      count
+    end
+
     # Helpers
 
     # Normalize whitespace in text

data/lib/llm_docs_builder/output_formatter.rb

@@ -3,8 +3,7 @@
 module LlmDocsBuilder
   # Formats output for CLI display
   #
-  # Provides formatting utilities for displaying comparison results,
-  # byte sizes, and numbers in a user-friendly way.
+  # Provides formatting utilities for displaying comparison results, byte sizes, and numbers in a user-friendly way.
   #
   # @api private
   class OutputFormatter
@@ -30,7 +29,7 @@ module LlmDocsBuilder
 
     # Format number with comma separators for readability
     #
-    # @param number [Integer] integer value
+    # @param number [Integer] value to format with comma separators
     # @return [String] formatted number with commas
     #
     # @example

data/lib/llm_docs_builder/parser.rb

@@ -3,9 +3,8 @@
 module LlmDocsBuilder
   # Parses llms.txt files into structured data
   #
-  # Reads and parses llms.txt files according to the llms.txt specification,
-  # extracting the title, description, and structured sections (Documentation,
-  # Examples, Optional) with their links.
+  # Reads and parses llms.txt files according to the llms.txt specification, extracting the title, description, and
+  # structured sections (Documentation, Examples, Optional) with their links.
   #
   # @example Parse an llms.txt file
   #   parser = LlmDocsBuilder::Parser.new('llms.txt')
@@ -109,8 +108,7 @@ module LlmDocsBuilder
 
   # Represents parsed llms.txt content with structured access to sections
   #
-  # Provides convenient access to parsed llms.txt sections including title,
-  # description, and link collections.
+  # Provides convenient access to parsed llms.txt sections including title, description, and link collections.
   #
   # @example Access parsed content
   #   parsed.title              # => "My Project"

data/lib/llm_docs_builder/text_compressor.rb

@@ -3,9 +3,8 @@
 module LlmDocsBuilder
   # Advanced text compression techniques for reducing token count
   #
-  # Provides more aggressive text compression methods including stopword removal,
-  # duplicate content detection, and sentence deduplication. These methods are more
-  # aggressive than basic markdown cleanup and should be used carefully.
+  # Provides more aggressive text compression methods including stopword removal, duplicate content detection, and
+  # sentence deduplication. These methods are more aggressive than basic markdown cleanup and should be used carefully.
   #
   # @example Basic usage
   #   compressor = LlmDocsBuilder::TextCompressor.new
@@ -72,8 +71,8 @@ module LlmDocsBuilder
     # Removes common English stopwords that don't carry significant meaning.
     # Preserves code blocks, inline code, and technical terms.
     #
-    # WARNING: This is an aggressive optimization that may affect readability.
-    # Use with caution and test results carefully.
+    # @deprecated This is an aggressive optimization that may affect readability.
+    #   Use with caution and test results carefully.
     #
     # @param content [String] text to process
     # @return [String] text with stopwords removed

data/lib/llm_docs_builder/token_estimator.rb

@@ -3,9 +3,8 @@
 module LlmDocsBuilder
   # Estimates token count for text content using character-based approximation
   #
-  # Provides token estimation without requiring external tokenizer dependencies.
-  # Uses the common heuristic that ~4 characters equals 1 token for English text,
-  # which works reasonably well for documentation and markdown content.
+  # Provides token estimation without requiring external tokenizer dependencies. Uses the common heuristic that ~4
+  # characters equals 1 token for English text, which works reasonably well for documentation and markdown content.
   #
   # @example Basic usage
   #   estimator = LlmDocsBuilder::TokenEstimator.new

data/lib/llm_docs_builder/transformers/base_transformer.rb

@@ -3,9 +3,8 @@
 module LlmDocsBuilder
   # Provides content transformation functionality
   #
-  # This module contains specialized transformers for modifying markdown content,
-  # including cleanup operations, link processing, heading normalization, and
-  # content enhancement for AI consumption.
+  # This module contains specialized transformers for modifying markdown content, including cleanup operations, link
+  # processing, heading normalization, and content enhancement for AI consumption.
   #
   # @api private
   module Transformers

data/lib/llm_docs_builder/transformers/heading_transformer.rb

@@ -4,9 +4,8 @@ module LlmDocsBuilder
   module Transformers
     # Normalizes headings to include hierarchical context
     #
-    # Transforms markdown headings to include parent context, making each section
-    # self-contained for RAG systems. This is particularly useful when documents
-    # are chunked and retrieved independently.
+    # Transforms markdown headings to include parent context, making each section self-contained for RAG systems. This
+    # is particularly useful when documents are chunked and retrieved independently.
     #
     # @example Basic heading normalization
     #   # Configuration
@@ -51,24 +50,36 @@ module LlmDocsBuilder
           next line if in_code_block
 
           # Match markdown headings (1-6 hash symbols followed by space and text)
-          heading_match = line.match(/^(#+)\s+(.+)$/)
+          # Supports optional ATX closing hashes (e.g., "## Title ##")
+          heading_match = line.match(/^(#+)\s+(.+?)(?:\s+#+)?\s*$/)
 
-          if heading_match && heading_match[1].count('#').between?(1, 6)
-            level = heading_match[1].count('#')
+          if heading_match && heading_match[1].length.between?(1, 6)
+            level = heading_match[1].length
             title = heading_match[2].strip
 
             # Update heading stack to current level
             heading_stack = heading_stack[0...level - 1]
-            heading_stack << title
+
+            # Compute the effective heading level. When same-level headings
+            # are nested under a parent (e.g., two consecutive ## headings),
+            # the child must receive a deeper markdown level so the output
+            # hierarchy is correct.
+            effective_level = if heading_stack.empty?
+                               level
+                             else
+                               [heading_stack.last[:effective_level] + 1, level].max
+                             end
+            effective_level = [effective_level, 6].min
+
+            heading_stack << { title: title, effective_level: effective_level }
 
             # Build hierarchical heading
             if level == 1
               # H1 stays as-is (top level)
               line
             else
-              # H2+ gets parent context
-              hierarchical_title = heading_stack.join(separator)
-              "#{'#' * level} #{hierarchical_title}\n"
+              hierarchical_title = heading_stack.map { |e| e[:title] }.join(separator)
+              "#{'#' * effective_level} #{hierarchical_title}\n"
             end
           else
             line

data/lib/llm_docs_builder/transformers/link_transformer.rb

@@ -4,8 +4,7 @@ module LlmDocsBuilder
   module Transformers
     # Transformer for link-related operations
     #
-    # Handles expansion of relative links to absolute URLs and
-    # conversion of HTML URLs to markdown format.
+    # Handles expansion of relative links to absolute URLs and conversion of HTML URLs to markdown format.
     #
     # @api public
     class LinkTransformer
@@ -30,6 +29,10 @@ module LlmDocsBuilder
 
       private
 
+      # URI schemes that must never be treated as relative paths when expanding
+      # links with a base URL (kept in sync with the converter's safe schemes)
+      ABSOLUTE_URL_SCHEMES = %w[http https mailto ftp tel].freeze
+
       # Expand relative links to absolute URLs
       #
       # @param content [String] markdown content
@@ -40,7 +43,7 @@ module LlmDocsBuilder
           text = ::Regexp.last_match(1)
           url = ::Regexp.last_match(2)
 
-          if url.start_with?('http://', 'https://', '//', '#')
+          if url.start_with?('http://', 'https://', '//', '#') || absolute_url_scheme?(url)
             match
           else
             clean_url = url.gsub(%r{^\./}, '')
@@ -50,6 +53,15 @@ module LlmDocsBuilder
         end
       end
 
+      # Check whether a link target uses a known absolute URI scheme
+      #
+      # @param url [String] link target
+      # @return [Boolean] true when the target already carries its own scheme
+      def absolute_url_scheme?(url)
+        scheme = url.split(':', 2).first.to_s
+        ABSOLUTE_URL_SCHEMES.include?(scheme)
+      end
+
       # Convert HTML URLs to markdown format
       #
       # @param content [String] markdown content

data/lib/llm_docs_builder/transformers/whitespace_transformer.rb

@@ -4,8 +4,7 @@ module LlmDocsBuilder
   module Transformers
     # Transformer for whitespace normalization
     #
-    # Reduces excessive blank lines and trailing whitespace to make
-    # content more compact for LLM consumption.
+    # Reduces excessive blank lines and trailing whitespace to make content more compact for LLM consumption.
     #
     # @api public
     class WhitespaceTransformer

data/lib/llm_docs_builder/validator.rb

@@ -56,8 +56,7 @@ module LlmDocsBuilder
 
     # Validate content and return result
     #
-    # Runs all validation checks, populates {#errors} array, and returns whether
-    # the content is valid.
+    # Runs all validation checks, populates {#errors} array, and returns whether the content is valid.
     #
     # @return [Boolean] true if content is valid, false otherwise
     def validate!

data/lib/llm_docs_builder/version.rb

@@ -2,5 +2,5 @@
 
 module LlmDocsBuilder
   # Current version of the LlmDocsBuilder gem
-  VERSION = '0.12.0'
+  VERSION = '1.0.0'
 end

data/lib/llm_docs_builder.rb

@@ -13,9 +13,8 @@ loader.setup
 
 # Build and optimize documentation for LLMs
 #
-# This gem provides tools for generating llms.txt files and transforming markdown
-# documentation to be AI-friendly. It can reduce token consumption by 67-95% while
-# preserving essential documentation content.
+# This gem provides tools for generating llms.txt files and transforming markdown documentation to be AI-friendly. It
+# can reduce token consumption by 67-95% while preserving essential documentation content.
 #
 # @api public
 module LlmDocsBuilder

data/llm-docs-builder.gemspec

@@ -20,13 +20,14 @@ Gem::Specification.new do |spec|
 
   spec.homepage = 'https://github.com/mensfeld/llm-docs-builder'
   spec.license = 'MIT'
-  spec.required_ruby_version = '>= 3.2'
+  spec.required_ruby_version = '>= 3.3'
 
   spec.metadata['allowed_push_host'] = 'https://rubygems.org'
   spec.metadata['homepage_uri'] = spec.homepage
   spec.metadata['source_code_uri'] = 'https://github.com/mensfeld/llm-docs-builder'
   spec.metadata['changelog_uri'] = 'https://github.com/mensfeld/llm-docs-builder/blob/master/CHANGELOG.md'
   spec.metadata['documentation_uri'] = 'https://github.com/mensfeld/llm-docs-builder'
+  spec.metadata['bug_tracker_uri'] = 'https://github.com/mensfeld/llm-docs-builder/issues'
   spec.metadata['rubygems_mfa_required'] = 'true'
 
   spec.files = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(spec|test)/}) }
@@ -38,9 +39,9 @@ Gem::Specification.new do |spec|
   spec.add_dependency 'nokogiri', '~> 1.17'
   spec.add_dependency 'zeitwerk', '~> 2.6'
 
-  spec.add_development_dependency 'bundler', '~> 2.0'
+  spec.add_development_dependency 'bundler'
   spec.add_development_dependency 'rake', '~> 13.0'
   spec.add_development_dependency 'rspec', '~> 3.0'
   spec.add_development_dependency 'rubocop', '~> 1.0'
-  spec.add_development_dependency 'simplecov', '~> 0.21'
+  spec.add_development_dependency 'simplecov', '~> 1.0'
 end