llm-docs-builder 0.3.0 → 0.7.0

data/lib/llm_docs_builder/text_compressor.rb

@@ -0,0 +1,164 @@
+# frozen_string_literal: true
+
+module LlmDocsBuilder
+  # Advanced text compression techniques for reducing token count
+  #
+  # Provides more aggressive text compression methods including stopword removal,
+  # duplicate content detection, and sentence deduplication. These methods are more
+  # aggressive than basic markdown cleanup and should be used carefully.
+  #
+  # @example Basic usage
+  #   compressor = LlmDocsBuilder::TextCompressor.new
+  #   compressed = compressor.compress("Your text here", remove_stopwords: true)
+  #
+  # @api public
+  class TextCompressor
+    # Common English stopwords that can be safely removed from documentation
+    # Excludes words that might be important in technical contexts (like "not", "no")
+    STOPWORDS = %w[
+      a an the this that these those
+      is am are was were be being been
+      have has had do does did
+      will would shall should may might must can could
+      i me my mine we us our ours
+      you your yours
+      he him his she her hers it its
+      they them their theirs
+      what which who whom whose where when why how
+      all both each few more most other some such
+      and or but if then else
+      at by for from in into of on to with
+      as so than
+      very really quite
+      there here
+      about above across after against along among around because before behind below
+      beneath beside besides between beyond during except inside near off since through
+      throughout under until up upon within without
+    ].freeze
+
+    # @return [Hash] compression options
+    attr_reader :options
+
+    # Initialize a new text compressor
+    #
+    # @param options [Hash] compression options
+    # @option options [Array<String>] :custom_stopwords additional stopwords to remove
+    # @option options [Boolean] :preserve_technical preserve technical terms and code
+    def initialize(options = {})
+      @options = {
+        preserve_technical: true,
+        custom_stopwords: []
+      }.merge(options)
+    end
+
+    # Compress text using configured methods
+    #
+    # @param content [String] text to compress
+    # @param methods [Hash] compression methods to apply
+    # @option methods [Boolean] :remove_stopwords remove common filler words
+    # @option methods [Boolean] :remove_duplicates remove duplicate sentences/paragraphs
+    # @return [String] compressed text
+    def compress(content, methods = {})
+      result = content.dup
+
+      result = remove_stopwords(result) if methods[:remove_stopwords]
+      result = remove_duplicate_paragraphs(result) if methods[:remove_duplicates]
+
+      result
+    end
+
+    # Remove stopwords from text while preserving technical content
+    #
+    # Removes common English stopwords that don't carry significant meaning.
+    # Preserves code blocks, inline code, and technical terms.
+    #
+    # WARNING: This is an aggressive optimization that may affect readability.
+    # Use with caution and test results carefully.
+    #
+    # @param content [String] text to process
+    # @return [String] text with stopwords removed
+    def remove_stopwords(content)
+      # Preserve code blocks by temporarily replacing them
+      code_blocks = {}
+      code_counter = 0
+
+      # Extract and preserve fenced code blocks
+      content = content.gsub(/^```.*?^```/m) do |match|
+        placeholder = "___CODE_BLOCK_#{code_counter}___"
+        code_blocks[placeholder] = match
+        code_counter += 1
+        placeholder
+      end
+
+      # Extract and preserve inline code
+      content = content.gsub(/`[^`]+`/) do |match|
+        placeholder = "___INLINE_CODE_#{code_counter}___"
+        code_blocks[placeholder] = match
+        code_counter += 1
+        placeholder
+      end
+
+      # Get combined stopwords list
+      stopwords_list = STOPWORDS + options[:custom_stopwords]
+
+      # Process each line
+      content = content.split("\n").map do |line|
+        # Skip markdown headers, lists, and links
+        if line.match?(/^#+\s/) || line.match?(/^[\*\-]\s/) || line.match?(/\[[^\]]+\]\([^)]+\)/)
+          line
+        else
+          # Remove stopwords from regular text
+          words = line.split(/\b/)
+          words.map do |word|
+            # Preserve the word if it's not a stopword or if we should preserve technical terms
+            if stopwords_list.include?(word.downcase) && !word.match?(/^[A-Z]/) # Don't remove capitalized words
+              ''
+            else
+              word
+            end
+          end.join
+        end
+      end.join("\n")
+
+      # Restore code blocks
+      code_blocks.each do |placeholder, original|
+        content = content.gsub(placeholder, original)
+      end
+
+      content
+    end
+
+    # Remove duplicate paragraphs from text
+    #
+    # Detects and removes paragraphs that are duplicates or near-duplicates.
+    # Documentation often repeats concepts across different sections.
+    #
+    # @param content [String] text to process
+    # @return [String] text with duplicate paragraphs removed
+    def remove_duplicate_paragraphs(content)
+      # Split into paragraphs (separated by blank lines)
+      paragraphs = content.split(/\n\s*\n/)
+
+      # Track seen paragraphs with normalized comparison
+      seen = {}
+      unique_paragraphs = []
+
+      paragraphs.each do |para|
+        # Normalize for comparison (remove extra whitespace, lowercase)
+        normalized = para.gsub(/\s+/, ' ').strip.downcase
+
+        # Skip empty paragraphs
+        next if normalized.empty?
+
+        # Check if we've seen this or similar paragraph
+        unless seen[normalized]
+          seen[normalized] = true
+          unique_paragraphs << para
+        end
+      end
+
+      unique_paragraphs.join("\n\n")
+    end
+
+  end
+end

data/lib/llm_docs_builder/token_estimator.rb

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+module LlmDocsBuilder
+  # Estimates token count for text content using character-based approximation
+  #
+  # Provides token estimation without requiring external tokenizer dependencies.
+  # Uses the common heuristic that ~4 characters equals 1 token for English text,
+  # which works reasonably well for documentation and markdown content.
+  #
+  # @example Basic usage
+  #   estimator = LlmDocsBuilder::TokenEstimator.new
+  #   token_count = estimator.estimate("This is a sample text.")
+  #
+  # @example With custom characters per token
+  #   estimator = LlmDocsBuilder::TokenEstimator.new(chars_per_token: 3.5)
+  #   token_count = estimator.estimate(content)
+  #
+  # @api public
+  class TokenEstimator
+    # Default number of characters per token
+    DEFAULT_CHARS_PER_TOKEN = 4.0
+
+    # @return [Float] characters per token ratio
+    attr_reader :chars_per_token
+
+    # Initialize a new token estimator
+    #
+    # @param chars_per_token [Float] number of characters per token (default: 4.0)
+    def initialize(chars_per_token: DEFAULT_CHARS_PER_TOKEN)
+      @chars_per_token = chars_per_token.to_f
+    end
+
+    # Estimate token count for given content
+    #
+    # @param content [String] text content to estimate tokens for
+    # @return [Integer] estimated number of tokens
+    def estimate(content)
+      return 0 if content.nil? || content.empty?
+
+      (content.length / chars_per_token).round
+    end
+
+    # Estimate token count (class method for convenience)
+    #
+    # @param content [String] text content to estimate tokens for
+    # @param chars_per_token [Float] number of characters per token (default: 4.0)
+    # @return [Integer] estimated number of tokens
+    def self.estimate(content, chars_per_token: DEFAULT_CHARS_PER_TOKEN)
+      new(chars_per_token: chars_per_token).estimate(content)
+    end
+  end
+end

data/lib/llm_docs_builder/transformers/base_transformer.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+module LlmDocsBuilder
+  module Transformers
+    # Base module for all transformers
+    #
+    # Provides a common interface for content transformation operations.
+    # Each transformer should implement the `transform` method.
+    #
+    # @api public
+    module BaseTransformer
+      # Transform content
+      #
+      # @param content [String] content to transform
+      # @param options [Hash] transformation options
+      # @return [String] transformed content
+      def transform(content, options = {})
+        raise NotImplementedError, "#{self.class} must implement #transform"
+      end
+
+      # Check if transformation should be applied
+      #
+      # @param options [Hash] transformation options
+      # @return [Boolean] true if transformation should be applied
+      def should_transform?(options)
+        true
+      end
+    end
+  end
+end

data/lib/llm_docs_builder/transformers/content_cleanup_transformer.rb

@@ -0,0 +1,106 @@
+# frozen_string_literal: true
+
+module LlmDocsBuilder
+  module Transformers
+    # Transformer for content cleanup operations
+    #
+    # Handles removal of various markdown elements that don't provide
+    # value for LLM consumption (frontmatter, comments, badges, etc.).
+    #
+    # @api public
+    class ContentCleanupTransformer
+      include BaseTransformer
+
+      # Transform content by removing unwanted elements
+      #
+      # @param content [String] markdown content
+      # @param options [Hash] transformation options
+      # @option options [Boolean] :remove_frontmatter remove YAML/TOML frontmatter
+      # @option options [Boolean] :remove_comments remove HTML comments
+      # @option options [Boolean] :remove_badges remove badge images
+      # @option options [Boolean] :remove_code_examples remove code blocks
+      # @option options [Boolean] :remove_images remove image syntax
+      # @option options [Boolean] :remove_blockquotes remove blockquote formatting
+      # @return [String] transformed content
+      def transform(content, options = {})
+        result = content.dup
+
+        result = remove_frontmatter(result) if options[:remove_frontmatter]
+        result = remove_comments(result) if options[:remove_comments]
+        result = remove_badges(result) if options[:remove_badges]
+        result = remove_code_examples(result) if options[:remove_code_examples]
+        result = remove_images(result) if options[:remove_images]
+        result = remove_blockquotes(result) if options[:remove_blockquotes]
+
+        result
+      end
+
+      private
+
+      # Remove YAML or TOML frontmatter
+      #
+      # @param content [String] markdown content
+      # @return [String] content without frontmatter
+      def remove_frontmatter(content)
+        content = content.sub(/\A---\s*$.*?^---\s*$/m, '')
+        content = content.sub(/\A\+\+\+\s*$.*?^\+\+\+\s*$/m, '')
+        content
+      end
+
+      # Remove HTML comments
+      #
+      # @param content [String] markdown content
+      # @return [String] content without comments
+      def remove_comments(content)
+        content.gsub(/<!--.*?-->/m, '')
+      end
+
+      # Remove badge images
+      #
+      # @param content [String] markdown content
+      # @return [String] content without badges
+      def remove_badges(content)
+        # Remove linked badges
+        content = content.gsub(/\[\!\[([^\]]*)\]\([^\)]*(?:badge|shield|svg|travis|coveralls|fury)[^\)]*\)\]\([^\)]*\)/i, '')
+        # Remove standalone badges
+        content = content.gsub(/!\[([^\]]*)\]\([^\)]*(?:badge|shield|svg|travis|coveralls|fury)[^\)]*\)/i, '')
+        content
+      end
+
+      # Remove code blocks and inline code
+      #
+      # @param content [String] markdown content
+      # @return [String] content without code
+      def remove_code_examples(content)
+        # Remove fenced code blocks
+        content = content.gsub(/^```.*?^```/m, '')
+        content = content.gsub(/^~~~.*?^~~~/m, '')
+        # Remove indented code blocks
+        content = content.gsub(/^([ ]{4,}|\t).+$/m, '')
+        # Remove inline code
+        content = content.gsub(/`[^`]+`/, '')
+        content
+      end
+
+      # Remove image syntax
+      #
+      # @param content [String] markdown content
+      # @return [String] content without images
+      def remove_images(content)
+        # Remove inline images
+        content = content.gsub(/!\[([^\]]*)\]\([^\)]+\)/, '')
+        # Remove reference-style images
+        content = content.gsub(/!\[([^\]]*)\]\[[^\]]+\]/, '')
+        content
+      end
+
+      # Remove blockquote formatting
+      #
+      # @param content [String] markdown content
+      # @return [String] content without blockquote markers
+      def remove_blockquotes(content)
+        content.gsub(/^>\s?/, '')
+      end
+    end
+  end
+end

data/lib/llm_docs_builder/transformers/enhancement_transformer.rb

@@ -0,0 +1,95 @@
+# frozen_string_literal: true
+
+module LlmDocsBuilder
+  module Transformers
+    # Transformer for document enhancements
+    #
+    # Adds helpful features like table of contents and custom instructions
+    # to improve LLM navigation and context understanding.
+    #
+    # @api public
+    class EnhancementTransformer
+      include BaseTransformer
+
+      # Transform content by adding enhancements
+      #
+      # @param content [String] markdown content
+      # @param options [Hash] transformation options
+      # @option options [Boolean] :generate_toc generate table of contents
+      # @option options [String] :custom_instruction custom instruction text
+      # @option options [Boolean] :remove_blockquotes whether blockquotes are being removed
+      # @return [String] transformed content
+      def transform(content, options = {})
+        result = content.dup
+
+        if options[:custom_instruction]
+          result = inject_custom_instruction(result, options[:custom_instruction], options[:remove_blockquotes])
+        end
+        result = generate_table_of_contents(result) if options[:generate_toc]
+
+        result
+      end
+
+      private
+
+      # Generate table of contents from headings
+      #
+      # @param content [String] markdown content
+      # @return [String] content with TOC prepended
+      def generate_table_of_contents(content)
+        headings = []
+        content.scan(/^(#{Regexp.escape('#')}{1,6})\s+(.+)$/) do
+          level = ::Regexp.last_match(1).length
+          title = ::Regexp.last_match(2).strip
+
+          anchor = title.downcase
+                        .gsub(/[^\w\s-]/, '')
+                        .gsub(/\s+/, '-')
+
+          headings << { level: level, title: title, anchor: anchor }
+        end
+
+        return content if headings.empty?
+
+        toc = ["## Table of Contents\n"]
+
+        headings.each do |heading|
+          next if heading[:level] == 1 && headings.first == heading
+
+          indent = '  ' * (heading[:level] - 1)
+          toc << "#{indent}- [#{heading[:title]}](##{heading[:anchor]})"
+        end
+
+        toc << "\n---\n"
+
+        if content.match(/^#\s+(.+)$/)
+          content.sub(/^(#\s+.+\n)/, "\\1\n#{toc.join("\n")}\n")
+        else
+          "#{toc.join("\n")}\n\n#{content}"
+        end
+      end
+
+      # Inject custom instruction at document top
+      #
+      # @param content [String] markdown content
+      # @param instruction [String] instruction text
+      # @param remove_blockquotes [Boolean] whether to avoid blockquote formatting
+      # @return [String] content with instruction prepended
+      def inject_custom_instruction(content, instruction, remove_blockquotes = false)
+        return content if instruction.nil? || instruction.empty?
+
+        formatted_instruction = if remove_blockquotes
+                                  "**AI Context**: #{instruction}\n\n---\n\n"
+                                else
+                                  "> **AI Context**: #{instruction}\n\n---\n\n"
+                                end
+
+        if content.match(/^#\s+(.+?)\n/)
+          content.sub(/^(#\s+.+?\n)/, "\\1\n#{formatted_instruction}")
+        else
+          "#{formatted_instruction}#{content}"
+        end
+      end
+    end
+  end
+end

data/lib/llm_docs_builder/transformers/link_transformer.rb

@@ -0,0 +1,84 @@
+# frozen_string_literal: true
+
+module LlmDocsBuilder
+  module Transformers
+    # Transformer for link-related operations
+    #
+    # Handles expansion of relative links to absolute URLs and
+    # conversion of HTML URLs to markdown format.
+    #
+    # @api public
+    class LinkTransformer
+      include BaseTransformer
+
+      # Transform links in content
+      #
+      # @param content [String] markdown content
+      # @param options [Hash] transformation options
+      # @option options [String] :base_url base URL for expanding relative links
+      # @option options [Boolean] :convert_urls convert HTML URLs to markdown format
+      # @return [String] transformed content
+      def transform(content, options = {})
+        result = content.dup
+
+        result = expand_relative_links(result, options[:base_url]) if options[:base_url]
+        result = convert_html_urls(result) if options[:convert_urls]
+        result = simplify_links(result) if options[:simplify_links]
+
+        result
+      end
+
+      private
+
+      # Expand relative links to absolute URLs
+      #
+      # @param content [String] markdown content
+      # @param base_url [String] base URL for expansion
+      # @return [String] content with expanded links
+      def expand_relative_links(content, base_url)
+        content.gsub(/\[([^\]]+)\]\(([^)]+)\)/) do |match|
+          text = ::Regexp.last_match(1)
+          url = ::Regexp.last_match(2)
+
+          if url.start_with?('http://', 'https://', '//', '#')
+            match
+          else
+            clean_url = url.gsub(%r{^\./}, '')
+            expanded_url = File.join(base_url, clean_url)
+            "[#{text}](#{expanded_url})"
+          end
+        end
+      end
+
+      # Convert HTML URLs to markdown format
+      #
+      # @param content [String] markdown content
+      # @return [String] content with converted URLs
+      def convert_html_urls(content)
+        content.gsub(%r{https?://[^\s<>]+\.html?(?=[)\s]|$)}) do |url|
+          url.sub(/\.html?$/, '.md')
+        end
+      end
+
+      # Simplify verbose link text
+      #
+      # @param content [String] markdown content
+      # @return [String] content with simplified links
+      def simplify_links(content)
+        content.gsub(/\[([^\]]+)\]\(([^)]+)\)/) do
+          text = ::Regexp.last_match(1)
+          url = ::Regexp.last_match(2)
+
+          simplified_text = text
+                            .gsub(/^(click here to|see|read more about|check out|visit)\s+(the\s+)?/i, '')
+                            .gsub(/\s+(here|documentation|docs)$/i, '')
+                            .strip
+
+          simplified_text = text if simplified_text.empty?
+
+          "[#{simplified_text}](#{url})"
+        end
+      end
+    end
+  end
+end

data/lib/llm_docs_builder/transformers/whitespace_transformer.rb

@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+module LlmDocsBuilder
+  module Transformers
+    # Transformer for whitespace normalization
+    #
+    # Reduces excessive blank lines and trailing whitespace to make
+    # content more compact for LLM consumption.
+    #
+    # @api public
+    class WhitespaceTransformer
+      include BaseTransformer
+
+      # Transform content by normalizing whitespace
+      #
+      # @param content [String] markdown content
+      # @param options [Hash] transformation options
+      # @option options [Boolean] :normalize_whitespace enable normalization
+      # @return [String] transformed content
+      def transform(content, options = {})
+        return content unless options[:normalize_whitespace]
+
+        normalize_whitespace(content)
+      end
+
+      private
+
+      # Normalize excessive whitespace
+      #
+      # @param content [String] markdown content
+      # @return [String] content with normalized whitespace
+      def normalize_whitespace(content)
+        # Remove trailing whitespace from each line
+        content = content.gsub(/ +$/, '')
+
+        # Reduce multiple consecutive blank lines to maximum of 2
+        content = content.gsub(/\n{4,}/, "\n\n\n")
+
+        # Trim leading and trailing whitespace
+        content.strip
+      end
+    end
+  end
+end

data/lib/llm_docs_builder/version.rb

@@ -2,5 +2,5 @@
 
 module LlmDocsBuilder
   # Current version of the LlmDocsBuilder gem
-  VERSION = '0.3.0'
+  VERSION = '0.7.0'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: llm-docs-builder
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 0.7.0
 platform: ruby
 authors:
 - Maciej Mensfeld
@@ -113,10 +113,10 @@ files:
 - ".github/workflows/docker.yml"
 - ".github/workflows/push.yml"
 - ".gitignore"
+- ".rspec"
 - ".rubocop.yml"
 - ".ruby-version"
 - CHANGELOG.md
-- CLAUDE.md
 - Dockerfile
 - Gemfile
 - Gemfile.lock
@@ -133,11 +133,18 @@ files:
 - lib/llm_docs_builder/errors.rb
 - lib/llm_docs_builder/generator.rb
 - lib/llm_docs_builder/markdown_transformer.rb
+- lib/llm_docs_builder/output_formatter.rb
 - lib/llm_docs_builder/parser.rb
+- lib/llm_docs_builder/text_compressor.rb
+- lib/llm_docs_builder/token_estimator.rb
+- lib/llm_docs_builder/transformers/base_transformer.rb
+- lib/llm_docs_builder/transformers/content_cleanup_transformer.rb
+- lib/llm_docs_builder/transformers/enhancement_transformer.rb
+- lib/llm_docs_builder/transformers/link_transformer.rb
+- lib/llm_docs_builder/transformers/whitespace_transformer.rb
 - lib/llm_docs_builder/validator.rb
 - lib/llm_docs_builder/version.rb
 - llm-docs-builder.gemspec
-- llm-docs-builder.yml
 - llm-docs-builder.yml.example
 - renovate.json
 homepage: https://github.com/mensfeld/llm-docs-builder