llm-docs-builder 0.10.0 → 0.12.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e6407836216f436b728247009f614ea4ea5c2b4de0edf855717129460df4b309
-  data.tar.gz: 8a556fc0b6307529f5c615c05b082521bd021e9e9f34ca30c8bb21d22b21deb2
+  metadata.gz: ac257dad79f49ed6993f784f8a28ee1e996e735fef4581449ad521ea9414a5d4
+  data.tar.gz: 29e1d2d578d57ea6f17aafca070c61b6161b6313d6614f0e4f798933ceae082d
 SHA512:
-  metadata.gz: 3a0b657545415c35187fa1595f3fcc5bd5c27a0c1a292bf00635d3c6f14221ac0a254f008f83acc1f7351ef76c6649e6b0540ac585cd64a670b05ef725b0308e
-  data.tar.gz: 68e95142e374ebae3c292db724a9163c396fd423eaa04983bbdaefd6f6cbb0bc72822dfbc5af9fd8f357f1545320db70bf5a6c8a2413fc5f796e73195adcdd13
+  metadata.gz: f82216cca621e942c0e6ad3d92aba5d099159cc9c0d10c1d010a85e2a740511103cebd0198c0056195775064853e749472dcb7f0939b8d3fda7753d291a5b0da
+  data.tar.gz: 31aa5737e215439b11a2e79d793dabb9ff342206b660a2ecd846920bc2f6501c3d5910da4cdc52ecfcfa9f7b9acef14213b17936edd23d86808c0bcb2f391952

data/.github/workflows/ci.yml

@@ -54,6 +54,18 @@ jobs:
           GITHUB_COVERAGE: ${{ matrix.coverage }}
         run: bin/rspecs
 
+  yard-lint:
+    timeout-minutes: 5
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5
+      - name: Set up Ruby
+        uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: '3.4.7'
+          bundler-cache: true
+      - name: Run yard-lint
+        run: bundle exec yard-lint lib/
 
   ci-success:
     name: CI Success
@@ -61,6 +73,7 @@ jobs:
     if: always()
     needs:
       - specs
+      - yard-lint
     steps:
       - name: Check all jobs passed
         if: |

data/.github/workflows/docker.yml

@@ -31,7 +31,7 @@ jobs:
 
       - name: Docker meta
         id: meta
-        uses: docker/metadata-action@c1e51972afc2121e065aed6d45c65596fe445f3f # v5
+        uses: docker/metadata-action@318604b99e75e41977312d83839a89be02ca4893 # v5
         with:
           images: |
             mensfeld/llm-docs-builder
@@ -45,7 +45,7 @@ jobs:
             type=raw,value=latest,enable={{is_default_branch}}
 
       - name: Set up QEMU
-        uses: docker/setup-qemu-action@29109295f81e9208d7d86ff1c6c12d2833863392 # v3
+        uses: docker/setup-qemu-action@c7c53464625b32c7a7e944ae62b3e17d2b600130 # v3
 
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v3

data/.github/workflows/push.yml

@@ -24,7 +24,7 @@ jobs:
           fetch-depth: 0
 
       - name: Set up Ruby
-        uses: ruby/setup-ruby@ab177d40ee5483edb974554986f56b33477e21d0 # v1.265.0
+        uses: ruby/setup-ruby@d5126b9b3579e429dd52e51e68624dda2e05be25 # v1.267.0
         with:
           bundler-cache: false
 
@@ -32,4 +32,4 @@ jobs:
         run: |
           bundle install --jobs 4 --retry 3
 
-      - uses: rubygems/release-gem@a25424ba2ba8b387abc8ef40807c2c85b96cbe32 # v1.1.1
+      - uses: rubygems/release-gem@1c162a739e8b4cb21a676e97b087e8268d8fc40b # v1.1.2

data/.gitignore

@@ -10,6 +10,8 @@
 /test/version_tmp/
 /tmp/
 mise.toml
+.DS_Store
+.vscode/launch.json
 
 # Used by dotenv library to load environment variables.
 .env
@@ -64,3 +66,9 @@ llms.txt
 # Config files that might contain sensitive data
 llms-txt.yml
 .llms-txt.yml
+
+# AI coding agent
+AGENTS.md
+CLAUDE.md
+GEMINI.md
+

data/CHANGELOG.md

@@ -1,5 +1,18 @@
 # Changelog
 
+## 0.12.0 (2025-11-12)
+- [Feature] **HTML to Markdown Reverse Converter** — Added support for converting HTML content to markdown format.
+  - Enables processing of HTML documentation sources
+  - Integrates seamlessly with the transformer pipeline
+  - Useful for converting web-based docs to markdown for further processing
+  - By @Eric-Guo in PR #32.
+
+## 0.11.0 (2025-11-03)
+- [Feature] **Transform from URL** — The `transform` command now accepts a remote URL via `--url` and processes fetched content through the standard transformer pipeline.
+  - Example: `llm-docs-builder transform --url https://example.com/docs/page.html`
+  - Applies all configured transformations and output options identically to local files
+  - By @Eric-Guo and @codex in PR #28.
+
 ## 0.10.0 (2025-10-27)
 - [Feature] **llms.txt Specification Compliance** - Updated output format to fully comply with the llms.txt specification from llmstxt.org.
   - **Metadata Format**: Metadata now appears within the description field using parentheses and comma separators: `- [title](url): description (tokens:450, updated:2025-10-13, priority:high)`

data/Gemfile

@@ -7,4 +7,8 @@ gemspec
 group :development do
   gem 'pry'
   gem 'pry-byebug'
+  gem 'yard-lint'
+end
+
+group :test do
 end

data/Gemfile.lock

@@ -1,7 +1,8 @@
 PATH
   remote: .
   specs:
-    llm-docs-builder (0.10.0)
+    llm-docs-builder (0.12.0)
+      nokogiri (~> 1.17)
       zeitwerk (~> 2.6)
 
 GEM
@@ -10,41 +11,62 @@ GEM
     ast (2.4.3)
     byebug (12.0.0)
     coderay (1.1.3)
+    date (3.5.0)
     diff-lcs (1.6.2)
     docile (1.4.1)
-    json (2.13.2)
+    erb (5.1.3)
+    io-console (0.8.1)
+    irb (1.15.3)
+      pp (>= 0.6.0)
+      rdoc (>= 4.0.0)
+      reline (>= 0.4.2)
+    json (2.16.0)
     language_server-protocol (3.17.0.5)
     lint_roller (1.1.0)
     method_source (1.1.0)
+    nokogiri (1.18.10-x86_64-linux-gnu)
+      racc (~> 1.4)
     parallel (1.27.0)
-    parser (3.3.9.0)
+    parser (3.3.10.0)
       ast (~> 2.4.1)
       racc
-    prism (1.4.0)
+    pp (0.6.3)
+      prettyprint
+    prettyprint (0.2.0)
+    prism (1.6.0)
     pry (0.15.2)
       coderay (~> 1.1)
       method_source (~> 1.0)
     pry-byebug (3.11.0)
       byebug (~> 12.0)
       pry (>= 0.13, < 0.16)
+    psych (5.2.6)
+      date
+      stringio
     racc (1.8.1)
     rainbow (3.1.1)
-    rake (13.3.0)
-    regexp_parser (2.11.2)
-    rspec (3.13.1)
+    rake (13.3.1)
+    rdoc (6.15.1)
+      erb
+      psych (>= 4.0.0)
+      tsort
+    regexp_parser (2.11.3)
+    reline (0.6.3)
+      io-console (~> 0.5)
+    rspec (3.13.2)
       rspec-core (~> 3.13.0)
       rspec-expectations (~> 3.13.0)
       rspec-mocks (~> 3.13.0)
-    rspec-core (3.13.5)
+    rspec-core (3.13.6)
       rspec-support (~> 3.13.0)
     rspec-expectations (3.13.5)
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.13.0)
-    rspec-mocks (3.13.5)
+    rspec-mocks (3.13.7)
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.13.0)
-    rspec-support (3.13.5)
-    rubocop (1.80.0)
+    rspec-support (3.13.6)
+    rubocop (1.81.7)
       json (~> 2.3)
       language_server-protocol (~> 3.17.0.2)
       lint_roller (~> 1.1.0)
@@ -52,10 +74,10 @@ GEM
       parser (>= 3.3.0.2)
       rainbow (>= 2.2.2, < 4.0)
       regexp_parser (>= 2.9.3, < 3.0)
-      rubocop-ast (>= 1.46.0, < 2.0)
+      rubocop-ast (>= 1.47.1, < 2.0)
       ruby-progressbar (~> 1.7)
       unicode-display_width (>= 2.4.0, < 4.0)
-    rubocop-ast (1.46.0)
+    rubocop-ast (1.47.1)
       parser (>= 3.3.7.2)
       prism (~> 1.4)
     ruby-progressbar (1.13.0)
@@ -65,13 +87,19 @@ GEM
       simplecov_json_formatter (~> 0.1)
     simplecov-html (0.13.2)
     simplecov_json_formatter (0.1.4)
-    unicode-display_width (3.1.5)
-      unicode-emoji (~> 4.0, >= 4.0.4)
-    unicode-emoji (4.0.4)
+    stringio (3.1.8)
+    tsort (0.2.0)
+    unicode-display_width (3.2.0)
+      unicode-emoji (~> 4.1)
+    unicode-emoji (4.1.0)
+    yard (0.9.37)
+    yard-lint (1.1.0)
+      irb
+      yard (~> 0.9)
+      zeitwerk (~> 2.6)
     zeitwerk (2.7.3)
 
 PLATFORMS
-  ruby
   x86_64-linux
 
 DEPENDENCIES
@@ -83,6 +111,7 @@ DEPENDENCIES
   rspec (~> 3.0)
   rubocop (~> 1.0)
   simplecov (~> 0.21)
+  yard-lint
 
 BUNDLED WITH
-   2.7.1
+   2.7.2

data/README.md

@@ -61,10 +61,15 @@ Factor:         2.8x smaller
 # Single file
 llm-docs-builder transform --docs README.md
 
+# Fetch and transform a remote page
+llm-docs-builder transform --url https://yoursite.com/docs/page.html
+
 # Bulk transform with config
 llm-docs-builder bulk-transform --config llm-docs-builder.yml
 ```
 
+**HTML to Markdown Conversion:** The transformer automatically detects and converts HTML content to clean markdown format. This works seamlessly with both local files and remote URLs, converting HTML tables, code blocks, and other elements into their markdown equivalents.
+
 ## Installation
 
 ### Docker (Recommended)
@@ -82,6 +87,20 @@ gem install llm-docs-builder
 
 ## Features
 
+### Automatic HTML to Markdown Conversion
+
+The tool automatically detects and converts HTML content to clean markdown:
+- **HTML Tables** → Markdown tables
+- **HTML Code Blocks** → Fenced code blocks
+- **Figures & Captions** → Clean markdown equivalents
+- **Seamless Integration** - Works with local files and remote URLs without special configuration
+
+```bash
+# Transform HTML content automatically
+llm-docs-builder transform --docs page-with-html.md
+llm-docs-builder transform --url https://site.com/docs/api.html
+```
+
 ### Measure and Compare
 
 ```bash

data/lib/llm_docs_builder/cli.rb

@@ -68,8 +68,9 @@ module LlmDocsBuilder
     # @param argv [Array<String>] command-line arguments
     # @return [Hash] parsed options including :command, :config, :docs, :output, :verbose
     def parse_options(argv)
+      command_token = argv.first
       options = {
-        command: argv.first&.match?(/^[a-z-]+$/) ? argv.shift : nil
+        command: command_token&.match?(/\A[a-z](?:[a-z-]*[a-z])?\z/) ? argv.shift : nil
       }
 
       OptionParser.new do |opts|
@@ -100,7 +101,7 @@ module LlmDocsBuilder
           options[:output] = path
         end
 
-        opts.on('-u', '--url URL', 'URL to fetch for comparison') do |url|
+        opts.on('-u', '--url URL', 'URL to fetch for transform or comparison') do |url|
           options[:url] = url
         end
 
@@ -185,21 +186,42 @@ module LlmDocsBuilder
       config = LlmDocsBuilder::Config.new(options[:config])
       merged_options = config.merge_with_options(options)
 
-      file_path = merged_options[:docs]
+      url = options[:url]
+      cli_file_path = options[:docs]
+      config_file_path = config['docs']
+      file_path = url ? cli_file_path : (cli_file_path || config_file_path)
 
-      unless file_path
-        puts 'File path required for transform command (use -d/--docs)'
+      if url && cli_file_path
+        puts 'Cannot use both --docs and --url for transform command'
         exit 1
       end
 
-      unless File.exist?(file_path)
-        puts "File not found: #{file_path}"
-        exit 1
+      unless file_path
+        unless url
+          puts 'File path required for transform command (use -d/--docs)'
+          exit 1
+        end
       end
 
-      puts "Transforming #{file_path}..." if merged_options[:verbose]
+      content =
+        if url
+          puts "Fetching #{url}..." if merged_options[:verbose]
+          fetcher = LlmDocsBuilder::UrlFetcher.new(verbose: merged_options[:verbose])
+          remote_content = fetcher.fetch(url)
+          puts "Transforming content from #{url}..." if merged_options[:verbose]
+          transform_options = merged_options.merge(content: remote_content, docs: nil, source_url: url)
+          LlmDocsBuilder.transform_markdown(nil, transform_options)
+        else
+          unless File.exist?(file_path)
+            puts "File not found: #{file_path}"
+            exit 1
+          end
 
-      content = LlmDocsBuilder.transform_markdown(file_path, merged_options)
+          puts "Transforming #{file_path}..." if merged_options[:verbose]
+
+          merged_options[:docs] = file_path
+          LlmDocsBuilder.transform_markdown(file_path, merged_options)
+        end
 
       if merged_options[:output] && merged_options[:output] != 'llms.txt'
         File.write(merged_options[:output], content)

data/lib/llm_docs_builder/comparator.rb

@@ -1,8 +1,5 @@
 # frozen_string_literal: true
 
-require 'net/http'
-require 'uri'
-
 module LlmDocsBuilder
   # Compares content sizes between human and AI versions
   #
@@ -30,7 +27,7 @@ module LlmDocsBuilder
     AI_USER_AGENT = 'Claude-Web/1.0 (Anthropic AI Assistant)'
 
     # Maximum number of redirects to follow before raising an error
-    MAX_REDIRECTS = 10
+    MAX_REDIRECTS = UrlFetcher::MAX_REDIRECTS
 
     # @return [String] URL to compare
     attr_reader :url
@@ -133,78 +130,11 @@ module LlmDocsBuilder
     # @return [String] response body
     # @raise [Errors::GenerationError] if fetch fails or too many redirects
     def fetch_url(url_string, user_agent, redirect_count = 0)
-      if redirect_count >= MAX_REDIRECTS
-        raise(
-          Errors::GenerationError,
-          "Too many redirects (#{MAX_REDIRECTS}) when fetching #{url_string}"
-        )
-      end
-
-      uri = validate_and_parse_url(url_string)
-
-      http = Net::HTTP.new(uri.host, uri.port)
-      http.use_ssl = uri.scheme == 'https'
-      http.open_timeout = 10
-      http.read_timeout = 30
-
-      request = Net::HTTP::Get.new(uri.request_uri)
-      request['User-Agent'] = user_agent
-
-      response = http.request(request)
-
-      case response
-      when Net::HTTPSuccess
-        response.body
-      when Net::HTTPRedirection
-        # Follow redirect with incremented counter
-        redirect_url = response['location']
-        puts "  Redirecting to #{redirect_url}..." if options[:verbose] && redirect_count.positive?
-        fetch_url(redirect_url, user_agent, redirect_count + 1)
-      else
-        raise(
-          Errors::GenerationError,
-          "Failed to fetch #{url_string}: #{response.code} #{response.message}"
-        )
-      end
-    rescue Errors::GenerationError
-      raise
-    rescue StandardError => e
-      raise(
-        Errors::GenerationError,
-        "Error fetching #{url_string}: #{e.message}"
-      )
-    end
-
-    # Validates and parses URL to prevent malformed URLs
-    #
-    # @param url_string [String] URL to validate and parse
-    # @return [URI::HTTP, URI::HTTPS] parsed URI
-    # @raise [Errors::GenerationError] if URL is invalid or uses unsupported scheme
-    def validate_and_parse_url(url_string)
-      uri = URI.parse(url_string)
-
-      # Only allow HTTP and HTTPS schemes
-      unless %w[http https].include?(uri.scheme&.downcase)
-        raise(
-          Errors::GenerationError,
-          "Unsupported URL scheme: #{uri.scheme || 'none'} (only http/https allowed)"
-        )
-      end
-
-      # Ensure host is present
-      if uri.host.nil? || uri.host.empty?
-        raise(
-          Errors::GenerationError,
-          "Invalid URL: missing host in #{url_string}"
-        )
-      end
-
-      uri
-    rescue URI::InvalidURIError => e
-      raise(
-        Errors::GenerationError,
-        "Invalid URL format: #{e.message}"
+      fetcher = UrlFetcher.new(
+        user_agent: user_agent,
+        verbose: options[:verbose]
       )
+      fetcher.fetch(url_string, redirect_count)
     end
 
     # Calculate comparison statistics

data/lib/llm_docs_builder/config.rb

@@ -53,11 +53,48 @@ module LlmDocsBuilder
     # defaults for any options not specified via CLI.
     #
     # @param options [Hash] CLI options hash
+    # @option options [String] :docs path to documentation directory or file
+    # @option options [String] :base_url base URL for expanding relative links
+    # @option options [String] :title project title
+    # @option options [String] :description project description
+    # @option options [String] :body additional body content
+    # @option options [String] :output output file path
+    # @option options [Boolean] :convert_urls convert HTML URLs to markdown format
+    # @option options [Boolean] :remove_comments remove HTML comments
+    # @option options [Boolean] :normalize_whitespace normalize whitespace
+    # @option options [Boolean] :remove_badges remove badge images
+    # @option options [Boolean] :remove_frontmatter remove YAML/TOML frontmatter
+    # @option options [Boolean] :verbose enable verbose output
+    # @option options [String] :suffix suffix for transformed files
+    # @option options [Array<String>] :excludes glob patterns for files to exclude
+    # @option options [Boolean] :bulk enable bulk transformation mode
+    # @option options [Boolean] :include_hidden include hidden files
+    # @option options [Boolean] :remove_code_examples remove code blocks
+    # @option options [Boolean] :remove_images remove image syntax
+    # @option options [Boolean] :simplify_links simplify link text
+    # @option options [Boolean] :remove_blockquotes remove blockquote formatting
+    # @option options [Boolean] :generate_toc generate table of contents
+    # @option options [String] :custom_instruction custom instruction text
+    # @option options [Boolean] :remove_stopwords remove common stopwords
+    # @option options [Boolean] :remove_duplicates remove duplicate paragraphs
+    # @option options [Boolean] :normalize_headings normalize heading hierarchy
+    # @option options [String] :heading_separator separator for heading paths
+    # @option options [Boolean] :include_metadata include metadata in output
+    # @option options [Boolean] :include_tokens include token counts
+    # @option options [Boolean] :include_timestamps include timestamps
+    # @option options [Boolean] :include_priority include priority metadata
+    # @option options [Boolean] :calculate_compression calculate compression ratios
+    # @option options [String] :content raw markdown content
+    # @option options [String] :source_url source URL for content
     # @return [Hash] merged configuration with CLI overrides applied
     def merge_with_options(options)
       # CLI options override config file, config file provides defaults
       {
-        docs: options[:docs] || self['docs'] || '.',
+        docs: if options.key?(:docs)
+                options[:docs]
+              else
+                self['docs'] || '.'
+              end,
         base_url: options[:base_url] || self['base_url'],
         title: options[:title] || self['title'],
         description: options[:description] || self['description'],
@@ -171,7 +208,10 @@ module LlmDocsBuilder
                                else
                                  self['calculate_compression'] || false
                                end
-      }
+      }.tap do |merged|
+        merged[:content] = options[:content] if options.key?(:content)
+        merged[:source_url] = options[:source_url] if options.key?(:source_url)
+      end
     end
 
     # Check if a config file was found and exists

data/lib/llm_docs_builder/helpers/prune_trailing_unsafe_link_separator.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+module LlmDocsBuilder
+  # Helper methods for content transformation
+  #
+  # @api private
+  module Helpers
+    # Removes trailing pipe characters and whitespace from array of string parts
+    #
+    # @param parts [Array<String>] array of string parts to process
+    # @return [void]
+    def prune_trailing_unsafe_link_separator!(parts)
+      while parts.any?
+        last = parts.last
+        new_last = last.sub(/[ \t]*\|\s*\z/, '')
+
+        if new_last != last
+          trimmed = new_last.rstrip
+          parts[-1] = trimmed
+          parts.pop if trimmed.empty?
+        elsif last.strip.empty?
+          parts.pop
+        else
+          break
+        end
+      end
+    end
+
+    module_function :prune_trailing_unsafe_link_separator!
+  end
+end

data/lib/llm_docs_builder/helpers/squeeze_blank_lines_outside_fences.rb

@@ -0,0 +1,71 @@
+# frozen_string_literal: true
+
+module LlmDocsBuilder
+  module Helpers
+    # Reduces consecutive blank lines outside of code fences
+    #
+    # @param text [String] input text to process
+    # @param max_blank [Integer] maximum number of consecutive blank lines to allow
+    # @param fence_chars [Array<String>] characters that can be used for code fences
+    # @param min_fence [Integer] minimum length of fence character sequence
+    # @return [String] processed text with squeezed blank lines
+    def squeeze_blank_lines_outside_fences(text, max_blank: 2, fence_chars: %w[` ~], min_fence: 3)
+      return '' if text.to_s.empty?
+
+      lines = text.split("\n", -1)
+
+      inside_fence = false
+      fence_indent = ''.dup
+      fence_char   = nil
+      fence_len    = 0
+
+      # Build a fast “does this look like an opening fence?” regex
+      # e.g., leading spaces + ``` or ~~~ (length >= min_fence) + optional info string
+      fence_set = Regexp.escape(fence_chars.join)
+      open_re   = /\A(\s*)([#{fence_set}])\2{#{min_fence - 1},}.*\z/
+
+      out = []
+      blank_streak = 0
+
+      lines.each_with_index do |line, _idx|
+        if inside_fence
+          out << line
+          # Closing fence must match indent, char, and fence length
+          if line.match?(/\A#{Regexp.escape(fence_indent)}#{Regexp.escape(fence_char * fence_len)}\s*\z/)
+            inside_fence = false
+            fence_indent = ''.dup
+            fence_char   = nil
+            fence_len    = 0
+          end
+          next
+        end
+
+        if (m = line.match(open_re))
+          # Enter fenced block; compute the *actual* fence length from the line
+          fence_indent = m[1]
+          fence_char   = m[2]
+          after_indent = line[fence_indent.length..]
+          fence_len    = after_indent[/\A#{Regexp.escape(fence_char)}+/].length
+          inside_fence = true
+          blank_streak = 0
+          out << line
+          next
+        end
+
+        # Outside fences: squeeze blank lines
+        if line.strip.empty?
+          blank_streak += 1
+          # Keep at most max_blank blank lines; skip extras
+          out << line if blank_streak <= max_blank
+        else
+          blank_streak = 0
+          out << line
+        end
+      end
+
+      out.join("\n")
+    end
+
+    module_function :squeeze_blank_lines_outside_fences
+  end
+end

data/lib/llm_docs_builder/helpers.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+module LlmDocsBuilder
+  module Helpers
+  end
+end
+
+require_relative 'helpers/squeeze_blank_lines_outside_fences'
+require_relative 'helpers/prune_trailing_unsafe_link_separator'